{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import tensorflow as tf\n",
    "\n",
    "import os\n",
    "import pickle\n",
    "import re\n",
    "from tensorflow.python.ops import math_ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlretrieve\n",
    "from os.path import isfile, isdir\n",
    "from tqdm import tqdm\n",
    "import zipfile\n",
    "import hashlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _unzip(save_path, _, database_name, data_path):\n",
    "    \"\"\"\n",
    "    Unzip wrapper with the same interface as _ungzip\n",
    "    :param save_path: The path of the gzip files\n",
    "    :param database_name: Name of database\n",
    "    :param data_path: Path to extract to\n",
    "    :param _: HACK - Used to have to same interface as _ungzip\n",
    "    \"\"\"\n",
    "    print('Extracting {}...'.format(database_name))\n",
    "    with zipfile.ZipFile(save_path) as zf:\n",
    "        zf.extractall(data_path)\n",
    "\n",
    "def download_extract(database_name, data_path):\n",
    "    \"\"\"\n",
    "    Download and extract database\n",
    "    :param database_name: Database name\n",
    "    \"\"\"\n",
    "    DATASET_ML1M = 'ml-1m'\n",
    "\n",
    "    if database_name == DATASET_ML1M:\n",
    "        url = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip'\n",
    "        hash_code = 'c4d9eecfca2ab87c1945afe126590906'\n",
    "        extract_path = os.path.join(data_path, 'ml-1m')\n",
    "        save_path = os.path.join(data_path, 'ml-1m.zip')\n",
    "        extract_fn = _unzip\n",
    "\n",
    "    if os.path.exists(extract_path):\n",
    "        print('Found {} Data'.format(database_name))\n",
    "        return\n",
    "\n",
    "    if not os.path.exists(data_path):\n",
    "        os.makedirs(data_path)\n",
    "\n",
    "    if not os.path.exists(save_path):\n",
    "        with DLProgress(unit='B', unit_scale=True, miniters=1, desc='Downloading {}'.format(database_name)) as pbar:\n",
    "            urlretrieve(\n",
    "                url,\n",
    "                save_path,\n",
    "                pbar.hook)\n",
    "\n",
    "    assert hashlib.md5(open(save_path, 'rb').read()).hexdigest() == hash_code, \\\n",
    "        '{} file is corrupted.  Remove the file and try again.'.format(save_path)\n",
    "\n",
    "    os.makedirs(extract_path)\n",
    "    try:\n",
    "        extract_fn(save_path, extract_path, database_name, data_path)\n",
    "    except Exception as err:\n",
    "        shutil.rmtree(extract_path)  # Remove extraction folder if there is an error\n",
    "        raise err\n",
    "\n",
    "    print('Done.')\n",
    "    # Remove compressed data\n",
    "#     os.remove(save_path)\n",
    "\n",
    "class DLProgress(tqdm):\n",
    "    \"\"\"\n",
    "    Handle Progress Bar while Downloading\n",
    "    \"\"\"\n",
    "    last_block = 0\n",
    "\n",
    "    def hook(self, block_num=1, block_size=1, total_size=None):\n",
    "        \"\"\"\n",
    "        A hook function that will be called once on establishment of the network connection and\n",
    "        once after each block read thereafter.\n",
    "        :param block_num: A count of blocks transferred so far\n",
    "        :param block_size: Block size in bytes\n",
    "        :param total_size: The total size of the file. This may be -1 on older FTP servers which do not return\n",
    "                            a file size in response to a retrieval request.\n",
    "        \"\"\"\n",
    "        self.total = total_size\n",
    "        self.update((block_num - self.last_block) * block_size)\n",
    "        self.last_block = block_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>OccupationID</th>\n",
       "      <th>Zip-code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>F</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>48067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>M</td>\n",
       "      <td>56</td>\n",
       "      <td>16</td>\n",
       "      <td>70072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>15</td>\n",
       "      <td>55117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>M</td>\n",
       "      <td>45</td>\n",
       "      <td>7</td>\n",
       "      <td>02460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>20</td>\n",
       "      <td>55455</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   UserID Gender  Age  OccupationID Zip-code\n",
       "0       1      F    1            10    48067\n",
       "1       2      M   56            16    70072\n",
       "2       3      M   25            15    55117\n",
       "3       4      M   45             7    02460\n",
       "4       5      M   25            20    55455"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users_title = ['UserID', 'Gender', 'Age', 'OccupationID', 'Zip-code']\n",
    "users = pd.read_table('./ml-1m/users.dat', sep='::', header=None, names=users_title, engine = 'python')\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MovieID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Animation|Children's|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MovieID                               Title                        Genres\n",
       "0        1                    Toy Story (1995)   Animation|Children's|Comedy\n",
       "1        2                      Jumanji (1995)  Adventure|Children's|Fantasy\n",
       "2        3             Grumpier Old Men (1995)                Comedy|Romance\n",
       "3        4            Waiting to Exhale (1995)                  Comedy|Drama\n",
       "4        5  Father of the Bride Part II (1995)                        Comedy"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_title = ['MovieID', 'Title', 'Genres']\n",
    "movies = pd.read_table('./ml-1m/movies.dat', sep='::', header=None, names=movies_title, engine = 'python')\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "评分数据\n",
    "分别有用户ID、电影ID、评分和时间戳等字段。\n",
    "\n",
    "数据中的格式：UserID::MovieID::Rating::Timestamp\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserID</th>\n",
       "      <th>MovieID</th>\n",
       "      <th>Rating</th>\n",
       "      <th>timestamps</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>661</td>\n",
       "      <td>3</td>\n",
       "      <td>978302109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>914</td>\n",
       "      <td>3</td>\n",
       "      <td>978301968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3408</td>\n",
       "      <td>4</td>\n",
       "      <td>978300275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2355</td>\n",
       "      <td>5</td>\n",
       "      <td>978824291</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   UserID  MovieID  Rating  timestamps\n",
       "0       1     1193       5   978300760\n",
       "1       1      661       3   978302109\n",
       "2       1      914       3   978301968\n",
       "3       1     3408       4   978300275\n",
       "4       1     2355       5   978824291"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_title= ['UserID','MovieID', 'Rating', 'timestamps']\n",
    "ratings = pd.read_table('./ml-1m/ratings.dat', sep='::', header=None, names=ratings_title, engine = 'python')\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data():\n",
    "    \"\"\"\n",
    "    Load Dataset from File\n",
    "    \"\"\"\n",
    "    #读取User数据\n",
    "    users_title = ['UserID', 'Gender', 'Age', 'JobID', 'Zip-code']\n",
    "    users = pd.read_table('./ml-1m/users.dat', sep='::', header=None, names=users_title, engine = 'python')\n",
    "    users = users.filter(regex='UserID|Gender|Age|JobID')\n",
    "    users_orig = users.values\n",
    "    #改变User数据中性别和年龄\n",
    "    gender_map = {'F':0, 'M':1}\n",
    "    users['Gender'] = users['Gender'].map(gender_map)\n",
    "\n",
    "    age_map = {val:ii for ii,val in enumerate(set(users['Age']))}\n",
    "    users['Age'] = users['Age'].map(age_map)\n",
    "\n",
    "    #读取Movie数据集\n",
    "    movies_title = ['MovieID', 'Title', 'Genres']\n",
    "    movies = pd.read_table('./ml-1m/movies.dat', sep='::', header=None, names=movies_title, engine = 'python')\n",
    "    movies_orig = movies.values\n",
    "    #将Title中的年份去掉\n",
    "    pattern = re.compile(r'^(.*)\\((\\d+)\\)$')\n",
    "\n",
    "    title_map = {val:pattern.match(val).group(1) for ii,val in enumerate(set(movies['Title']))}\n",
    "    movies['Title'] = movies['Title'].map(title_map)\n",
    "\n",
    "    #电影类型转数字字典\n",
    "    genres_set = set()\n",
    "    for val in movies['Genres'].str.split('|'):\n",
    "        genres_set.update(val)\n",
    "\n",
    "    genres_set.add('<PAD>')\n",
    "    genres2int = {val:ii for ii, val in enumerate(genres_set)}\n",
    "\n",
    "    #将电影类型转成等长数字列表，长度是18\n",
    "    genres_map = {val:[genres2int[row] for row in val.split('|')] for ii,val in enumerate(set(movies['Genres']))}\n",
    "\n",
    "    for key in genres_map:\n",
    "        for cnt in range(max(genres2int.values()) - len(genres_map[key])):\n",
    "            genres_map[key].insert(len(genres_map[key]) + cnt,genres2int['<PAD>'])\n",
    "    \n",
    "    movies['Genres'] = movies['Genres'].map(genres_map)\n",
    "\n",
    "    #电影Title转数字字典\n",
    "    title_set = set()\n",
    "    for val in movies['Title'].str.split():\n",
    "        title_set.update(val)\n",
    "    \n",
    "    title_set.add('<PAD>')\n",
    "    title2int = {val:ii for ii, val in enumerate(title_set)}\n",
    "\n",
    "    #将电影Title转成等长数字列表，长度是15\n",
    "    title_count = 15\n",
    "    title_map = {val:[title2int[row] for row in val.split()] for ii,val in enumerate(set(movies['Title']))}\n",
    "    \n",
    "    for key in title_map:\n",
    "        for cnt in range(title_count - len(title_map[key])):\n",
    "            title_map[key].insert(len(title_map[key]) + cnt,title2int['<PAD>'])\n",
    "    \n",
    "    movies['Title'] = movies['Title'].map(title_map)\n",
    "\n",
    "    #读取评分数据集\n",
    "    ratings_title = ['UserID','MovieID', 'ratings', 'timestamps']\n",
    "    ratings = pd.read_table('./ml-1m/ratings.dat', sep='::', header=None, names=ratings_title, engine = 'python')\n",
    "    ratings = ratings.filter(regex='UserID|MovieID|ratings')\n",
    "\n",
    "    #合并三个表\n",
    "    data = pd.merge(pd.merge(ratings, users), movies)\n",
    "    \n",
    "    #将数据分成X和y两张表\n",
    "    target_fields = ['ratings']\n",
    "    features_pd, targets_pd = data.drop(target_fields, axis=1), data[target_fields]\n",
    "    \n",
    "    features = features_pd.values\n",
    "    targets_values = targets_pd.values\n",
    "    \n",
    "    return title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig = load_data()\n",
    "\n",
    "pickle.dump((title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig), open('preprocess.p', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>JobID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   UserID  Gender  Age  JobID\n",
       "0       1       0    0     10\n",
       "1       2       1    5     16\n",
       "2       3       1    6     15\n",
       "3       4       1    2      7\n",
       "4       5       1    6     20"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MovieID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>[2527, 4617, 1434, 1434, 1434, 1434, 1434, 143...</td>\n",
       "      <td>[9, 10, 7, 16, 16, 16, 16, 16, 16, 16, 16, 16,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>[47, 1434, 1434, 1434, 1434, 1434, 1434, 1434,...</td>\n",
       "      <td>[14, 10, 15, 16, 16, 16, 16, 16, 16, 16, 16, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>[3706, 4698, 4401, 1434, 1434, 1434, 1434, 143...</td>\n",
       "      <td>[7, 5, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>[896, 4253, 3898, 1434, 1434, 1434, 1434, 1434...</td>\n",
       "      <td>[7, 6, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>[4468, 2727, 3264, 5186, 2078, 755, 1434, 1434...</td>\n",
       "      <td>[7, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MovieID                                              Title  \\\n",
       "0        1  [2527, 4617, 1434, 1434, 1434, 1434, 1434, 143...   \n",
       "1        2  [47, 1434, 1434, 1434, 1434, 1434, 1434, 1434,...   \n",
       "2        3  [3706, 4698, 4401, 1434, 1434, 1434, 1434, 143...   \n",
       "3        4  [896, 4253, 3898, 1434, 1434, 1434, 1434, 1434...   \n",
       "4        5  [4468, 2727, 3264, 5186, 2078, 755, 1434, 1434...   \n",
       "\n",
       "                                              Genres  \n",
       "0  [9, 10, 7, 16, 16, 16, 16, 16, 16, 16, 16, 16,...  \n",
       "1  [14, 10, 15, 16, 16, 16, 16, 16, 16, 16, 16, 1...  \n",
       "2  [7, 5, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,...  \n",
       "3  [7, 6, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,...  \n",
       "4  [7, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig = pickle.load(open('preprocess.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "def save_params(params):\n",
    "    \"\"\"\n",
    "    Save parameters to file\n",
    "    \"\"\"\n",
    "    pickle.dump(params, open('params.p', 'wb'))\n",
    "\n",
    "\n",
    "def load_params():\n",
    "    \"\"\"\n",
    "    Load parameters from file\n",
    "    \"\"\"\n",
    "    return pickle.load(open('params.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#嵌入矩阵的维度\n",
    "embed_dim = 32\n",
    "#用户ID个数\n",
    "uid_max = max(features.take(0,1)) + 1 # 6040\n",
    "#性别个数\n",
    "gender_max = max(features.take(2,1)) + 1 # 1 + 1 = 2\n",
    "#年龄类别个数\n",
    "age_max = max(features.take(3,1)) + 1 # 6 + 1 = 7\n",
    "#职业个数\n",
    "job_max = max(features.take(4,1)) + 1# 20 + 1 = 21\n",
    "\n",
    "#电影ID个数\n",
    "movie_id_max = max(features.take(1,1)) + 1 # 3952\n",
    "#电影类型个数\n",
    "movie_categories_max = max(genres2int.values()) + 1 # 18 + 1 = 19\n",
    "#电影名单词个数\n",
    "movie_title_max = len(title_set) # 5216\n",
    "\n",
    "#对电影类型嵌入向量做加和操作的标志，考虑过使用mean做平均，但是没实现mean\n",
    "combiner = \"sum\"\n",
    "\n",
    "#电影名长度\n",
    "sentences_size = title_count # = 15\n",
    "#文本卷积滑动窗口，分别滑动2, 3, 4, 5个单词\n",
    "window_sizes = {2, 3, 4, 5}\n",
    "#文本卷积核数量\n",
    "filter_num = 8\n",
    "\n",
    "#电影ID转下标的字典，数据集中电影ID跟下标不一致，比如第5行的数据电影ID不一定是5\n",
    "movieid2idx = {val[0]:i for i, val in enumerate(movies.values)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Epochs\n",
    "num_epochs = 5\n",
    "# Batch Size\n",
    "batch_size = 256\n",
    "\n",
    "dropout_keep = 0.5\n",
    "# Learning Rate\n",
    "learning_rate = 0.0001\n",
    "# Show stats for every n number of batches\n",
    "show_every_n_batches = 20\n",
    "\n",
    "save_dir = './save'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "输入\n",
    "定义输入的占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_inputs():\n",
    "    uid = tf.placeholder(tf.int32, [None, 1], name=\"uid\")\n",
    "    user_gender = tf.placeholder(tf.int32, [None, 1], name=\"user_gender\")\n",
    "    user_age = tf.placeholder(tf.int32, [None, 1], name=\"user_age\")\n",
    "    user_job = tf.placeholder(tf.int32, [None, 1], name=\"user_job\")\n",
    "    \n",
    "    movie_id = tf.placeholder(tf.int32, [None, 1], name=\"movie_id\")\n",
    "    movie_categories = tf.placeholder(tf.int32, [None, 18], name=\"movie_categories\")\n",
    "    movie_titles = tf.placeholder(tf.int32, [None, 15], name=\"movie_titles\")\n",
    "    targets = tf.placeholder(tf.int32, [None, 1], name=\"targets\")\n",
    "    LearningRate = tf.placeholder(tf.float32, name = \"LearningRate\")\n",
    "    dropout_keep_prob = tf.placeholder(tf.float32, name = \"dropout_keep_prob\")\n",
    "    return uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, LearningRate, dropout_keep_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_user_embedding(uid, user_gender, user_age, user_job):\n",
    "    with tf.name_scope(\"user_embedding\"):\n",
    "        uid_embed_matrix = tf.Variable(tf.random_uniform([uid_max, embed_dim], -1, 1), name = \"uid_embed_matrix\")\n",
    "        uid_embed_layer = tf.nn.embedding_lookup(uid_embed_matrix, uid, name = \"uid_embed_layer\")\n",
    "    \n",
    "        gender_embed_matrix = tf.Variable(tf.random_uniform([gender_max, embed_dim // 2], -1, 1), name= \"gender_embed_matrix\")\n",
    "        gender_embed_layer = tf.nn.embedding_lookup(gender_embed_matrix, user_gender, name = \"gender_embed_layer\")\n",
    "        \n",
    "        age_embed_matrix = tf.Variable(tf.random_uniform([age_max, embed_dim // 2], -1, 1), name=\"age_embed_matrix\")\n",
    "        age_embed_layer = tf.nn.embedding_lookup(age_embed_matrix, user_age, name=\"age_embed_layer\")\n",
    "        \n",
    "        job_embed_matrix = tf.Variable(tf.random_uniform([job_max, embed_dim // 2], -1, 1), name = \"job_embed_matrix\")\n",
    "        job_embed_layer = tf.nn.embedding_lookup(job_embed_matrix, user_job, name = \"job_embed_layer\")\n",
    "    return uid_embed_layer, gender_embed_layer, age_embed_layer, job_embed_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_user_feature_layer(uid_embed_layer, gender_embed_layer, age_embed_layer, job_embed_layer):\n",
    "    with tf.name_scope(\"user_fc\"):\n",
    "        #第一层全连接\n",
    "        uid_fc_layer = tf.layers.dense(uid_embed_layer, embed_dim, name = \"uid_fc_layer\", activation=tf.nn.relu)\n",
    "        gender_fc_layer = tf.layers.dense(gender_embed_layer, embed_dim, name = \"gender_fc_layer\", activation=tf.nn.relu)\n",
    "        age_fc_layer = tf.layers.dense(age_embed_layer, embed_dim, name =\"age_fc_layer\", activation=tf.nn.relu)\n",
    "        job_fc_layer = tf.layers.dense(job_embed_layer, embed_dim, name = \"job_fc_layer\", activation=tf.nn.relu)\n",
    "        \n",
    "        #第二层全连接\n",
    "        user_combine_layer = tf.concat([uid_fc_layer, gender_fc_layer, age_fc_layer, job_fc_layer], 2)  #(?, 1, 128)\n",
    "        user_combine_layer = tf.contrib.layers.fully_connected(user_combine_layer, 200, tf.tanh)  #(?, 1, 200)\n",
    "    \n",
    "        user_combine_layer_flat = tf.reshape(user_combine_layer, [-1, 200])\n",
    "    return user_combine_layer, user_combine_layer_flat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movie_id_embed_layer(movie_id):\n",
    "    with tf.name_scope(\"movie_embedding\"):\n",
    "        movie_id_embed_matrix = tf.Variable(tf.random_uniform([movie_id_max, embed_dim], -1, 1), name = \"movie_id_embed_matrix\")\n",
    "        movie_id_embed_layer = tf.nn.embedding_lookup(movie_id_embed_matrix, movie_id, name = \"movie_id_embed_layer\")\n",
    "    return movie_id_embed_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movie_categories_layers(movie_categories):\n",
    "    with tf.name_scope(\"movie_categories_layers\"):\n",
    "        movie_categories_embed_matrix = tf.Variable(tf.random_uniform([movie_categories_max, embed_dim], -1, 1), name = \"movie_categories_embed_matrix\")\n",
    "        movie_categories_embed_layer = tf.nn.embedding_lookup(movie_categories_embed_matrix, movie_categories, name = \"movie_categories_embed_layer\")\n",
    "        if combiner == \"sum\":\n",
    "            movie_categories_embed_layer = tf.reduce_sum(movie_categories_embed_layer, axis=1, keep_dims=True)\n",
    "    #     elif combiner == \"mean\":\n",
    "\n",
    "    return movie_categories_embed_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movie_cnn_layer(movie_titles):\n",
    "    #从嵌入矩阵中得到电影名对应的各个单词的嵌入向量\n",
    "    with tf.name_scope(\"movie_embedding\"):\n",
    "        movie_title_embed_matrix = tf.Variable(tf.random_uniform([movie_title_max, embed_dim], -1, 1), name = \"movie_title_embed_matrix\")\n",
    "        movie_title_embed_layer = tf.nn.embedding_lookup(movie_title_embed_matrix, movie_titles, name = \"movie_title_embed_layer\")\n",
    "        movie_title_embed_layer_expand = tf.expand_dims(movie_title_embed_layer, -1)\n",
    "    \n",
    "    #对文本嵌入层使用不同尺寸的卷积核做卷积和最大池化\n",
    "    pool_layer_lst = []\n",
    "    for window_size in window_sizes:\n",
    "        with tf.name_scope(\"movie_txt_conv_maxpool_{}\".format(window_size)):\n",
    "            filter_weights = tf.Variable(tf.truncated_normal([window_size, embed_dim, 1, filter_num],stddev=0.1),name = \"filter_weights\")\n",
    "            filter_bias = tf.Variable(tf.constant(0.1, shape=[filter_num]), name=\"filter_bias\")\n",
    "            \n",
    "            conv_layer = tf.nn.conv2d(movie_title_embed_layer_expand, filter_weights, [1,1,1,1], padding=\"VALID\", name=\"conv_layer\")\n",
    "            relu_layer = tf.nn.relu(tf.nn.bias_add(conv_layer,filter_bias), name =\"relu_layer\")\n",
    "            \n",
    "            maxpool_layer = tf.nn.max_pool(relu_layer, [1,sentences_size - window_size + 1 ,1,1], [1,1,1,1], padding=\"VALID\", name=\"maxpool_layer\")\n",
    "            pool_layer_lst.append(maxpool_layer)\n",
    "\n",
    "    #Dropout层\n",
    "    with tf.name_scope(\"pool_dropout\"):\n",
    "        pool_layer = tf.concat(pool_layer_lst, 3, name =\"pool_layer\")\n",
    "        max_num = len(window_sizes) * filter_num\n",
    "        pool_layer_flat = tf.reshape(pool_layer , [-1, 1, max_num], name = \"pool_layer_flat\")\n",
    "    \n",
    "        dropout_layer = tf.nn.dropout(pool_layer_flat, dropout_keep_prob, name = \"dropout_layer\")\n",
    "    return pool_layer_flat, dropout_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movie_feature_layer(movie_id_embed_layer, movie_categories_embed_layer, dropout_layer):\n",
    "    with tf.name_scope(\"movie_fc\"):\n",
    "        #第一层全连接\n",
    "        movie_id_fc_layer = tf.layers.dense(movie_id_embed_layer, embed_dim, name = \"movie_id_fc_layer\", activation=tf.nn.relu)\n",
    "        movie_categories_fc_layer = tf.layers.dense(movie_categories_embed_layer, embed_dim, name = \"movie_categories_fc_layer\", activation=tf.nn.relu)\n",
    "    \n",
    "        #第二层全连接\n",
    "        movie_combine_layer = tf.concat([movie_id_fc_layer, movie_categories_fc_layer, dropout_layer], 2)  #(?, 1, 96)\n",
    "        movie_combine_layer = tf.contrib.layers.fully_connected(movie_combine_layer, 200, tf.tanh)  #(?, 1, 200)\n",
    "    \n",
    "        movie_combine_layer_flat = tf.reshape(movie_combine_layer, [-1, 200])\n",
    "    return movie_combine_layer, movie_combine_layer_flat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-19-559a1ee9ce9e>:6: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "train_graph = tf.Graph()\n",
    "with train_graph.as_default():\n",
    "    #获取输入占位符\n",
    "    uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob = get_inputs()\n",
    "    #获取User的4个嵌入向量\n",
    "    uid_embed_layer, gender_embed_layer, age_embed_layer, job_embed_layer = get_user_embedding(uid, user_gender, user_age, user_job)\n",
    "    #得到用户特征\n",
    "    user_combine_layer, user_combine_layer_flat = get_user_feature_layer(uid_embed_layer, gender_embed_layer, age_embed_layer, job_embed_layer)\n",
    "    #获取电影ID的嵌入向量\n",
    "    movie_id_embed_layer = get_movie_id_embed_layer(movie_id)\n",
    "    #获取电影类型的嵌入向量\n",
    "    movie_categories_embed_layer = get_movie_categories_layers(movie_categories)\n",
    "    #获取电影名的特征向量\n",
    "    pool_layer_flat, dropout_layer = get_movie_cnn_layer(movie_titles)\n",
    "    #得到电影特征\n",
    "    movie_combine_layer, movie_combine_layer_flat = get_movie_feature_layer(movie_id_embed_layer, \n",
    "                                                                                movie_categories_embed_layer, \n",
    "                                                                                dropout_layer)\n",
    "    #计算出评分，要注意两个不同的方案，inference的名字（name值）是不一样的，后面做推荐时要根据name取得tensor\n",
    "    with tf.name_scope(\"inference\"):\n",
    "        #将用户特征和电影特征作为输入，经过全连接，输出一个值的方案\n",
    "#         inference_layer = tf.concat([user_combine_layer_flat, movie_combine_layer_flat], 1)  #(?, 200)\n",
    "#         inference = tf.layers.dense(inference_layer, 1,\n",
    "#                                     kernel_initializer=tf.truncated_normal_initializer(stddev=0.01), \n",
    "#                                     kernel_regularizer=tf.nn.l2_loss, name=\"inference\")\n",
    "        #简单的将用户特征和电影特征做矩阵乘法得到一个预测评分\n",
    "#        inference = tf.matmul(user_combine_layer_flat, tf.transpose(movie_combine_layer_flat))\n",
    "        inference = tf.reduce_sum(user_combine_layer_flat * movie_combine_layer_flat, axis=1)\n",
    "        inference = tf.expand_dims(inference, axis=1)\n",
    "\n",
    "    with tf.name_scope(\"loss\"):\n",
    "        # MSE损失，将计算值回归到评分\n",
    "        cost = tf.losses.mean_squared_error(targets, inference )\n",
    "        loss = tf.reduce_mean(cost)\n",
    "    # 优化损失 \n",
    "#     train_op = tf.train.AdamOptimizer(lr).minimize(loss)  #cost\n",
    "    global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "    optimizer = tf.train.AdamOptimizer(lr)\n",
    "    gradients = optimizer.compute_gradients(loss)  #cost\n",
    "    train_op = optimizer.apply_gradients(gradients, global_step=global_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'inference/ExpandDims:0' shape=(?, 1) dtype=float32>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batches(Xs, ys, batch_size):\n",
    "    for start in range(0, len(Xs), batch_size):\n",
    "        end = min(start + batch_size, len(Xs))\n",
    "        yield Xs[start:end], ys[start:end]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing to F:\\JupyterProjects\\推荐系统\\runs\\1532524073\n",
      "\n",
      "2018-07-25T21:07:57.829940: Epoch   0 Batch    0/3125   train_loss = 12.481\n",
      "2018-07-25T21:07:58.967005: Epoch   0 Batch   20/3125   train_loss = 3.973\n",
      "2018-07-25T21:08:00.018065: Epoch   0 Batch   40/3125   train_loss = 2.786\n",
      "2018-07-25T21:08:01.124128: Epoch   0 Batch   60/3125   train_loss = 1.954\n",
      "2018-07-25T21:08:02.229192: Epoch   0 Batch   80/3125   train_loss = 2.065\n",
      "2018-07-25T21:08:03.302253: Epoch   0 Batch  100/3125   train_loss = 1.887\n",
      "2018-07-25T21:08:04.406316: Epoch   0 Batch  120/3125   train_loss = 1.829\n",
      "2018-07-25T21:08:05.545381: Epoch   0 Batch  140/3125   train_loss = 1.612\n",
      "2018-07-25T21:08:06.686447: Epoch   0 Batch  160/3125   train_loss = 1.419\n",
      "2018-07-25T21:08:07.863514: Epoch   0 Batch  180/3125   train_loss = 1.624\n",
      "2018-07-25T21:08:08.991578: Epoch   0 Batch  200/3125   train_loss = 1.603\n",
      "2018-07-25T21:08:10.118643: Epoch   0 Batch  220/3125   train_loss = 1.521\n",
      "2018-07-25T21:08:11.306711: Epoch   0 Batch  240/3125   train_loss = 1.394\n",
      "2018-07-25T21:08:12.436776: Epoch   0 Batch  260/3125   train_loss = 1.470\n",
      "2018-07-25T21:08:13.546839: Epoch   0 Batch  280/3125   train_loss = 1.395\n",
      "2018-07-25T21:08:14.750908: Epoch   0 Batch  300/3125   train_loss = 1.503\n",
      "2018-07-25T21:08:15.972978: Epoch   0 Batch  320/3125   train_loss = 1.432\n",
      "2018-07-25T21:08:17.103042: Epoch   0 Batch  340/3125   train_loss = 1.370\n",
      "2018-07-25T21:08:18.219106: Epoch   0 Batch  360/3125   train_loss = 1.360\n",
      "2018-07-25T21:08:19.439176: Epoch   0 Batch  380/3125   train_loss = 1.221\n",
      "2018-07-25T21:08:20.589242: Epoch   0 Batch  400/3125   train_loss = 1.220\n",
      "2018-07-25T21:08:21.756309: Epoch   0 Batch  420/3125   train_loss = 1.306\n",
      "2018-07-25T21:08:22.950377: Epoch   0 Batch  440/3125   train_loss = 1.522\n",
      "2018-07-25T21:08:24.127444: Epoch   0 Batch  460/3125   train_loss = 1.323\n",
      "2018-07-25T21:08:25.187505: Epoch   0 Batch  480/3125   train_loss = 1.375\n",
      "2018-07-25T21:08:26.329570: Epoch   0 Batch  500/3125   train_loss = 1.014\n",
      "2018-07-25T21:08:27.490637: Epoch   0 Batch  520/3125   train_loss = 1.276\n",
      "2018-07-25T21:08:28.595700: Epoch   0 Batch  540/3125   train_loss = 1.151\n",
      "2018-07-25T21:08:29.668761: Epoch   0 Batch  560/3125   train_loss = 1.328\n",
      "2018-07-25T21:08:30.762824: Epoch   0 Batch  580/3125   train_loss = 1.380\n",
      "2018-07-25T21:08:31.920890: Epoch   0 Batch  600/3125   train_loss = 1.411\n",
      "2018-07-25T21:08:33.087957: Epoch   0 Batch  620/3125   train_loss = 1.401\n",
      "2018-07-25T21:08:34.254023: Epoch   0 Batch  640/3125   train_loss = 1.331\n",
      "2018-07-25T21:08:35.438091: Epoch   0 Batch  660/3125   train_loss = 1.327\n",
      "2018-07-25T21:08:36.588157: Epoch   0 Batch  680/3125   train_loss = 1.216\n",
      "2018-07-25T21:08:37.764224: Epoch   0 Batch  700/3125   train_loss = 1.312\n",
      "2018-07-25T21:08:38.847286: Epoch   0 Batch  720/3125   train_loss = 1.194\n",
      "2018-07-25T21:08:39.996352: Epoch   0 Batch  740/3125   train_loss = 1.297\n",
      "2018-07-25T21:08:41.153418: Epoch   0 Batch  760/3125   train_loss = 1.345\n",
      "2018-07-25T21:08:42.267482: Epoch   0 Batch  780/3125   train_loss = 1.322\n",
      "2018-07-25T21:08:43.462550: Epoch   0 Batch  800/3125   train_loss = 1.177\n",
      "2018-07-25T21:08:44.587614: Epoch   0 Batch  820/3125   train_loss = 1.239\n",
      "2018-07-25T21:08:45.787683: Epoch   0 Batch  840/3125   train_loss = 1.198\n",
      "2018-07-25T21:08:46.954750: Epoch   0 Batch  860/3125   train_loss = 1.106\n",
      "2018-07-25T21:08:48.103816: Epoch   0 Batch  880/3125   train_loss = 1.156\n",
      "2018-07-25T21:08:49.334886: Epoch   0 Batch  900/3125   train_loss = 1.220\n",
      "2018-07-25T21:08:50.424948: Epoch   0 Batch  920/3125   train_loss = 1.249\n",
      "2018-07-25T21:08:51.519011: Epoch   0 Batch  940/3125   train_loss = 1.377\n",
      "2018-07-25T21:08:52.721080: Epoch   0 Batch  960/3125   train_loss = 1.313\n",
      "2018-07-25T21:08:53.865145: Epoch   0 Batch  980/3125   train_loss = 1.321\n",
      "2018-07-25T21:08:54.933206: Epoch   0 Batch 1000/3125   train_loss = 1.283\n",
      "2018-07-25T21:08:56.103273: Epoch   0 Batch 1020/3125   train_loss = 1.308\n",
      "2018-07-25T21:08:57.346344: Epoch   0 Batch 1040/3125   train_loss = 1.187\n",
      "2018-07-25T21:08:58.489410: Epoch   0 Batch 1060/3125   train_loss = 1.436\n",
      "2018-07-25T21:08:59.561471: Epoch   0 Batch 1080/3125   train_loss = 1.180\n",
      "2018-07-25T21:09:00.692536: Epoch   0 Batch 1100/3125   train_loss = 1.302\n",
      "2018-07-25T21:09:01.851602: Epoch   0 Batch 1120/3125   train_loss = 1.231\n",
      "2018-07-25T21:09:02.995667: Epoch   0 Batch 1140/3125   train_loss = 1.271\n",
      "2018-07-25T21:09:04.158734: Epoch   0 Batch 1160/3125   train_loss = 1.233\n",
      "2018-07-25T21:09:05.370803: Epoch   0 Batch 1180/3125   train_loss = 1.266\n",
      "2018-07-25T21:09:06.567872: Epoch   0 Batch 1200/3125   train_loss = 1.192\n",
      "2018-07-25T21:09:07.669935: Epoch   0 Batch 1220/3125   train_loss = 1.126\n",
      "2018-07-25T21:09:08.871003: Epoch   0 Batch 1240/3125   train_loss = 1.075\n",
      "2018-07-25T21:09:10.079072: Epoch   0 Batch 1260/3125   train_loss = 1.196\n",
      "2018-07-25T21:09:11.229138: Epoch   0 Batch 1280/3125   train_loss = 1.246\n",
      "2018-07-25T21:09:12.362203: Epoch   0 Batch 1300/3125   train_loss = 1.214\n",
      "2018-07-25T21:09:13.502268: Epoch   0 Batch 1320/3125   train_loss = 1.190\n",
      "2018-07-25T21:09:14.645334: Epoch   0 Batch 1340/3125   train_loss = 1.150\n",
      "2018-07-25T21:09:15.785399: Epoch   0 Batch 1360/3125   train_loss = 1.144\n",
      "2018-07-25T21:09:16.859460: Epoch   0 Batch 1380/3125   train_loss = 1.096\n",
      "2018-07-25T21:09:18.017527: Epoch   0 Batch 1400/3125   train_loss = 1.246\n",
      "2018-07-25T21:09:19.153592: Epoch   0 Batch 1420/3125   train_loss = 1.259\n",
      "2018-07-25T21:09:20.328659: Epoch   0 Batch 1440/3125   train_loss = 1.120\n",
      "2018-07-25T21:09:21.454723: Epoch   0 Batch 1460/3125   train_loss = 1.241\n",
      "2018-07-25T21:09:22.584788: Epoch   0 Batch 1480/3125   train_loss = 1.266\n",
      "2018-07-25T21:09:23.665850: Epoch   0 Batch 1500/3125   train_loss = 1.304\n",
      "2018-07-25T21:09:24.823916: Epoch   0 Batch 1520/3125   train_loss = 1.281\n",
      "2018-07-25T21:09:26.010984: Epoch   0 Batch 1540/3125   train_loss = 1.287\n",
      "2018-07-25T21:09:27.151049: Epoch   0 Batch 1560/3125   train_loss = 1.203\n",
      "2018-07-25T21:09:28.238111: Epoch   0 Batch 1580/3125   train_loss = 1.275\n",
      "2018-07-25T21:09:29.388177: Epoch   0 Batch 1600/3125   train_loss = 1.252\n",
      "2018-07-25T21:09:30.590246: Epoch   0 Batch 1620/3125   train_loss = 1.146\n",
      "2018-07-25T21:09:31.727311: Epoch   0 Batch 1640/3125   train_loss = 1.270\n",
      "2018-07-25T21:09:32.878377: Epoch   0 Batch 1660/3125   train_loss = 1.330\n",
      "2018-07-25T21:09:33.976439: Epoch   0 Batch 1680/3125   train_loss = 1.181\n",
      "2018-07-25T21:09:35.097503: Epoch   0 Batch 1700/3125   train_loss = 1.070\n",
      "2018-07-25T21:09:36.235569: Epoch   0 Batch 1720/3125   train_loss = 1.172\n",
      "2018-07-25T21:09:37.304630: Epoch   0 Batch 1740/3125   train_loss = 1.140\n",
      "2018-07-25T21:09:38.373691: Epoch   0 Batch 1760/3125   train_loss = 1.271\n",
      "2018-07-25T21:09:39.497755: Epoch   0 Batch 1780/3125   train_loss = 1.124\n",
      "2018-07-25T21:09:40.591818: Epoch   0 Batch 1800/3125   train_loss = 1.141\n",
      "2018-07-25T21:09:41.723882: Epoch   0 Batch 1820/3125   train_loss = 1.138\n",
      "2018-07-25T21:09:42.837946: Epoch   0 Batch 1840/3125   train_loss = 1.248\n",
      "2018-07-25T21:09:44.003013: Epoch   0 Batch 1860/3125   train_loss = 1.231\n",
      "2018-07-25T21:09:45.172080: Epoch   0 Batch 1880/3125   train_loss = 1.261\n",
      "2018-07-25T21:09:46.342147: Epoch   0 Batch 1900/3125   train_loss = 0.982\n",
      "2018-07-25T21:09:47.490212: Epoch   0 Batch 1920/3125   train_loss = 1.162\n",
      "2018-07-25T21:09:48.656279: Epoch   0 Batch 1940/3125   train_loss = 1.088\n",
      "2018-07-25T21:09:49.829346: Epoch   0 Batch 1960/3125   train_loss = 1.161\n",
      "2018-07-25T21:09:51.037415: Epoch   0 Batch 1980/3125   train_loss = 1.119\n",
      "2018-07-25T21:09:52.278486: Epoch   0 Batch 2000/3125   train_loss = 1.391\n",
      "2018-07-25T21:09:53.484555: Epoch   0 Batch 2020/3125   train_loss = 1.225\n",
      "2018-07-25T21:09:54.918637: Epoch   0 Batch 2040/3125   train_loss = 1.115\n",
      "2018-07-25T21:09:56.109705: Epoch   0 Batch 2060/3125   train_loss = 0.958\n",
      "2018-07-25T21:09:57.309774: Epoch   0 Batch 2080/3125   train_loss = 1.279\n",
      "2018-07-25T21:09:58.434838: Epoch   0 Batch 2100/3125   train_loss = 1.154\n",
      "2018-07-25T21:09:59.616906: Epoch   0 Batch 2120/3125   train_loss = 1.109\n",
      "2018-07-25T21:10:00.744970: Epoch   0 Batch 2140/3125   train_loss = 1.140\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-07-25T21:10:01.891036: Epoch   0 Batch 2160/3125   train_loss = 1.092\n",
      "2018-07-25T21:10:03.073104: Epoch   0 Batch 2180/3125   train_loss = 1.117\n",
      "2018-07-25T21:10:04.266172: Epoch   0 Batch 2200/3125   train_loss = 1.108\n",
      "2018-07-25T21:10:05.414237: Epoch   0 Batch 2220/3125   train_loss = 1.151\n",
      "2018-07-25T21:10:06.594305: Epoch   0 Batch 2240/3125   train_loss = 1.007\n",
      "2018-07-25T21:10:07.815375: Epoch   0 Batch 2260/3125   train_loss = 1.119\n",
      "2018-07-25T21:10:08.999443: Epoch   0 Batch 2280/3125   train_loss = 1.167\n",
      "2018-07-25T21:10:10.193511: Epoch   0 Batch 2300/3125   train_loss = 1.164\n",
      "2018-07-25T21:10:11.394580: Epoch   0 Batch 2320/3125   train_loss = 1.231\n",
      "2018-07-25T21:10:12.575647: Epoch   0 Batch 2340/3125   train_loss = 1.185\n",
      "2018-07-25T21:10:13.634708: Epoch   0 Batch 2360/3125   train_loss = 1.127\n",
      "2018-07-25T21:10:14.721770: Epoch   0 Batch 2380/3125   train_loss = 1.143\n",
      "2018-07-25T21:10:15.839834: Epoch   0 Batch 2400/3125   train_loss = 1.312\n",
      "2018-07-25T21:10:16.953897: Epoch   0 Batch 2420/3125   train_loss = 1.138\n",
      "2018-07-25T21:10:18.046960: Epoch   0 Batch 2440/3125   train_loss = 1.254\n",
      "2018-07-25T21:10:19.152023: Epoch   0 Batch 2460/3125   train_loss = 1.060\n",
      "2018-07-25T21:10:20.310089: Epoch   0 Batch 2480/3125   train_loss = 1.175\n",
      "2018-07-25T21:10:21.441154: Epoch   0 Batch 2500/3125   train_loss = 1.206\n",
      "2018-07-25T21:10:22.598220: Epoch   0 Batch 2520/3125   train_loss = 1.085\n",
      "2018-07-25T21:10:23.712284: Epoch   0 Batch 2540/3125   train_loss = 1.039\n",
      "2018-07-25T21:10:24.942354: Epoch   0 Batch 2560/3125   train_loss = 0.913\n",
      "2018-07-25T21:10:26.076419: Epoch   0 Batch 2580/3125   train_loss = 1.015\n",
      "2018-07-25T21:10:27.224485: Epoch   0 Batch 2600/3125   train_loss = 1.179\n",
      "2018-07-25T21:10:28.361550: Epoch   0 Batch 2620/3125   train_loss = 1.098\n",
      "2018-07-25T21:10:29.540617: Epoch   0 Batch 2640/3125   train_loss = 1.103\n",
      "2018-07-25T21:10:30.715685: Epoch   0 Batch 2660/3125   train_loss = 1.233\n",
      "2018-07-25T21:10:31.850750: Epoch   0 Batch 2680/3125   train_loss = 0.979\n",
      "2018-07-25T21:10:33.038817: Epoch   0 Batch 2700/3125   train_loss = 1.131\n",
      "2018-07-25T21:10:34.154881: Epoch   0 Batch 2720/3125   train_loss = 1.093\n",
      "2018-07-25T21:10:35.281946: Epoch   0 Batch 2740/3125   train_loss = 1.144\n",
      "2018-07-25T21:10:36.375008: Epoch   0 Batch 2760/3125   train_loss = 1.186\n",
      "2018-07-25T21:10:37.437069: Epoch   0 Batch 2780/3125   train_loss = 1.129\n",
      "2018-07-25T21:10:38.562133: Epoch   0 Batch 2800/3125   train_loss = 1.344\n",
      "2018-07-25T21:10:39.667197: Epoch   0 Batch 2820/3125   train_loss = 1.370\n",
      "2018-07-25T21:10:40.765259: Epoch   0 Batch 2840/3125   train_loss = 1.117\n",
      "2018-07-25T21:10:41.873323: Epoch   0 Batch 2860/3125   train_loss = 1.074\n",
      "2018-07-25T21:10:43.018388: Epoch   0 Batch 2880/3125   train_loss = 1.124\n",
      "2018-07-25T21:10:44.169454: Epoch   0 Batch 2900/3125   train_loss = 1.155\n",
      "2018-07-25T21:10:45.287518: Epoch   0 Batch 2920/3125   train_loss = 1.156\n",
      "2018-07-25T21:10:46.385581: Epoch   0 Batch 2940/3125   train_loss = 1.117\n",
      "2018-07-25T21:10:47.519646: Epoch   0 Batch 2960/3125   train_loss = 1.195\n",
      "2018-07-25T21:10:48.687713: Epoch   0 Batch 2980/3125   train_loss = 1.139\n",
      "2018-07-25T21:10:49.773775: Epoch   0 Batch 3000/3125   train_loss = 1.167\n",
      "2018-07-25T21:10:50.900839: Epoch   0 Batch 3020/3125   train_loss = 1.237\n",
      "2018-07-25T21:10:52.009903: Epoch   0 Batch 3040/3125   train_loss = 1.160\n",
      "2018-07-25T21:10:53.130967: Epoch   0 Batch 3060/3125   train_loss = 1.185\n",
      "2018-07-25T21:10:54.212029: Epoch   0 Batch 3080/3125   train_loss = 1.225\n",
      "2018-07-25T21:10:55.307091: Epoch   0 Batch 3100/3125   train_loss = 1.175\n",
      "2018-07-25T21:10:56.426155: Epoch   0 Batch 3120/3125   train_loss = 0.995\n",
      "2018-07-25T21:10:56.863180: Epoch   0 Batch    0/781   test_loss = 0.995\n",
      "2018-07-25T21:10:57.233201: Epoch   0 Batch   20/781   test_loss = 1.068\n",
      "2018-07-25T21:10:57.542219: Epoch   0 Batch   40/781   test_loss = 1.110\n",
      "2018-07-25T21:10:57.844236: Epoch   0 Batch   60/781   test_loss = 1.323\n",
      "2018-07-25T21:10:58.160254: Epoch   0 Batch   80/781   test_loss = 1.336\n",
      "2018-07-25T21:10:58.466272: Epoch   0 Batch  100/781   test_loss = 1.332\n",
      "2018-07-25T21:10:58.778290: Epoch   0 Batch  120/781   test_loss = 1.244\n",
      "2018-07-25T21:10:59.071306: Epoch   0 Batch  140/781   test_loss = 1.164\n",
      "2018-07-25T21:10:59.367323: Epoch   0 Batch  160/781   test_loss = 1.314\n",
      "2018-07-25T21:10:59.665340: Epoch   0 Batch  180/781   test_loss = 1.262\n",
      "2018-07-25T21:10:59.953357: Epoch   0 Batch  200/781   test_loss = 1.151\n",
      "2018-07-25T21:11:00.244374: Epoch   0 Batch  220/781   test_loss = 0.937\n",
      "2018-07-25T21:11:00.541391: Epoch   0 Batch  240/781   test_loss = 1.169\n",
      "2018-07-25T21:11:00.855409: Epoch   0 Batch  260/781   test_loss = 1.141\n",
      "2018-07-25T21:11:01.199428: Epoch   0 Batch  280/781   test_loss = 1.386\n",
      "2018-07-25T21:11:01.509446: Epoch   0 Batch  300/781   test_loss = 1.224\n",
      "2018-07-25T21:11:01.818464: Epoch   0 Batch  320/781   test_loss = 1.300\n",
      "2018-07-25T21:11:02.128481: Epoch   0 Batch  340/781   test_loss = 0.893\n",
      "2018-07-25T21:11:02.428498: Epoch   0 Batch  360/781   test_loss = 1.285\n",
      "2018-07-25T21:11:02.729516: Epoch   0 Batch  380/781   test_loss = 1.108\n",
      "2018-07-25T21:11:03.037533: Epoch   0 Batch  400/781   test_loss = 1.077\n",
      "2018-07-25T21:11:03.338551: Epoch   0 Batch  420/781   test_loss = 1.027\n",
      "2018-07-25T21:11:03.642568: Epoch   0 Batch  440/781   test_loss = 1.171\n",
      "2018-07-25T21:11:03.945585: Epoch   0 Batch  460/781   test_loss = 1.051\n",
      "2018-07-25T21:11:04.241602: Epoch   0 Batch  480/781   test_loss = 1.151\n",
      "2018-07-25T21:11:04.540619: Epoch   0 Batch  500/781   test_loss = 1.001\n",
      "2018-07-25T21:11:04.834636: Epoch   0 Batch  520/781   test_loss = 1.141\n",
      "2018-07-25T21:11:05.128653: Epoch   0 Batch  540/781   test_loss = 0.953\n",
      "2018-07-25T21:11:05.436671: Epoch   0 Batch  560/781   test_loss = 1.206\n",
      "2018-07-25T21:11:05.735688: Epoch   0 Batch  580/781   test_loss = 1.115\n",
      "2018-07-25T21:11:06.042705: Epoch   0 Batch  600/781   test_loss = 1.207\n",
      "2018-07-25T21:11:06.344722: Epoch   0 Batch  620/781   test_loss = 1.245\n",
      "2018-07-25T21:11:06.661741: Epoch   0 Batch  640/781   test_loss = 1.295\n",
      "2018-07-25T21:11:06.973758: Epoch   0 Batch  660/781   test_loss = 1.150\n",
      "2018-07-25T21:11:07.286776: Epoch   0 Batch  680/781   test_loss = 1.371\n",
      "2018-07-25T21:11:07.617795: Epoch   0 Batch  700/781   test_loss = 1.131\n",
      "2018-07-25T21:11:07.915812: Epoch   0 Batch  720/781   test_loss = 1.299\n",
      "2018-07-25T21:11:08.221830: Epoch   0 Batch  740/781   test_loss = 1.197\n",
      "2018-07-25T21:11:08.529847: Epoch   0 Batch  760/781   test_loss = 1.145\n",
      "2018-07-25T21:11:08.823864: Epoch   0 Batch  780/781   test_loss = 1.115\n",
      "2018-07-25T21:11:10.286948: Epoch   1 Batch   15/3125   train_loss = 1.195\n",
      "2018-07-25T21:11:11.457015: Epoch   1 Batch   35/3125   train_loss = 1.194\n",
      "2018-07-25T21:11:12.612081: Epoch   1 Batch   55/3125   train_loss = 1.297\n",
      "2018-07-25T21:11:13.763147: Epoch   1 Batch   75/3125   train_loss = 1.115\n",
      "2018-07-25T21:11:14.921213: Epoch   1 Batch   95/3125   train_loss = 0.977\n",
      "2018-07-25T21:11:16.014276: Epoch   1 Batch  115/3125   train_loss = 1.148\n",
      "2018-07-25T21:11:17.195343: Epoch   1 Batch  135/3125   train_loss = 1.045\n",
      "2018-07-25T21:11:18.344409: Epoch   1 Batch  155/3125   train_loss = 1.082\n",
      "2018-07-25T21:11:19.489474: Epoch   1 Batch  175/3125   train_loss = 1.016\n",
      "2018-07-25T21:11:20.707544: Epoch   1 Batch  195/3125   train_loss = 1.189\n",
      "2018-07-25T21:11:21.892612: Epoch   1 Batch  215/3125   train_loss = 1.122\n",
      "2018-07-25T21:11:23.063679: Epoch   1 Batch  235/3125   train_loss = 1.051\n",
      "2018-07-25T21:11:24.239746: Epoch   1 Batch  255/3125   train_loss = 1.194\n",
      "2018-07-25T21:11:25.436814: Epoch   1 Batch  275/3125   train_loss = 1.021\n",
      "2018-07-25T21:11:26.574880: Epoch   1 Batch  295/3125   train_loss = 1.018\n",
      "2018-07-25T21:11:27.752947: Epoch   1 Batch  315/3125   train_loss = 0.976\n",
      "2018-07-25T21:11:28.926014: Epoch   1 Batch  335/3125   train_loss = 0.966\n",
      "2018-07-25T21:11:30.092081: Epoch   1 Batch  355/3125   train_loss = 1.113\n",
      "2018-07-25T21:11:31.349153: Epoch   1 Batch  375/3125   train_loss = 1.157\n",
      "2018-07-25T21:11:32.458216: Epoch   1 Batch  395/3125   train_loss = 1.107\n",
      "2018-07-25T21:11:33.569280: Epoch   1 Batch  415/3125   train_loss = 1.252\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-07-25T21:11:34.716345: Epoch   1 Batch  435/3125   train_loss = 1.120\n",
      "2018-07-25T21:11:35.816408: Epoch   1 Batch  455/3125   train_loss = 1.088\n",
      "2018-07-25T21:11:36.987475: Epoch   1 Batch  475/3125   train_loss = 1.167\n",
      "2018-07-25T21:11:38.122540: Epoch   1 Batch  495/3125   train_loss = 1.052\n",
      "2018-07-25T21:11:39.265605: Epoch   1 Batch  515/3125   train_loss = 1.187\n",
      "2018-07-25T21:11:40.427672: Epoch   1 Batch  535/3125   train_loss = 1.095\n",
      "2018-07-25T21:11:41.607739: Epoch   1 Batch  555/3125   train_loss = 1.286\n",
      "2018-07-25T21:11:42.745805: Epoch   1 Batch  575/3125   train_loss = 1.101\n",
      "2018-07-25T21:11:43.972875: Epoch   1 Batch  595/3125   train_loss = 1.251\n",
      "2018-07-25T21:11:45.181944: Epoch   1 Batch  615/3125   train_loss = 1.077\n",
      "2018-07-25T21:11:46.398013: Epoch   1 Batch  635/3125   train_loss = 1.137\n",
      "2018-07-25T21:11:47.603082: Epoch   1 Batch  655/3125   train_loss = 0.966\n",
      "2018-07-25T21:11:48.776149: Epoch   1 Batch  675/3125   train_loss = 0.954\n",
      "2018-07-25T21:11:49.919215: Epoch   1 Batch  695/3125   train_loss = 1.063\n",
      "2018-07-25T21:11:51.056280: Epoch   1 Batch  715/3125   train_loss = 1.073\n",
      "2018-07-25T21:11:52.254348: Epoch   1 Batch  735/3125   train_loss = 0.953\n",
      "2018-07-25T21:11:53.353411: Epoch   1 Batch  755/3125   train_loss = 1.129\n",
      "2018-07-25T21:11:54.772492: Epoch   1 Batch  775/3125   train_loss = 1.014\n",
      "2018-07-25T21:11:55.974561: Epoch   1 Batch  795/3125   train_loss = 1.185\n",
      "2018-07-25T21:11:57.148628: Epoch   1 Batch  815/3125   train_loss = 1.134\n",
      "2018-07-25T21:11:58.348697: Epoch   1 Batch  835/3125   train_loss = 1.052\n",
      "2018-07-25T21:11:59.548766: Epoch   1 Batch  855/3125   train_loss = 1.395\n",
      "2018-07-25T21:12:00.640828: Epoch   1 Batch  875/3125   train_loss = 1.146\n",
      "2018-07-25T21:12:01.800894: Epoch   1 Batch  895/3125   train_loss = 1.024\n",
      "2018-07-25T21:12:02.958961: Epoch   1 Batch  915/3125   train_loss = 1.099\n",
      "2018-07-25T21:12:04.166030: Epoch   1 Batch  935/3125   train_loss = 1.161\n",
      "2018-07-25T21:12:05.382099: Epoch   1 Batch  955/3125   train_loss = 1.147\n",
      "2018-07-25T21:12:06.604169: Epoch   1 Batch  975/3125   train_loss = 1.084\n",
      "2018-07-25T21:12:07.829239: Epoch   1 Batch  995/3125   train_loss = 0.915\n",
      "2018-07-25T21:12:08.996306: Epoch   1 Batch 1015/3125   train_loss = 1.106\n",
      "2018-07-25T21:12:10.181374: Epoch   1 Batch 1035/3125   train_loss = 1.104\n",
      "2018-07-25T21:12:11.367442: Epoch   1 Batch 1055/3125   train_loss = 1.088\n",
      "2018-07-25T21:12:12.515507: Epoch   1 Batch 1075/3125   train_loss = 1.088\n",
      "2018-07-25T21:12:13.685574: Epoch   1 Batch 1095/3125   train_loss = 0.997\n",
      "2018-07-25T21:12:14.816639: Epoch   1 Batch 1115/3125   train_loss = 1.096\n",
      "2018-07-25T21:12:15.912702: Epoch   1 Batch 1135/3125   train_loss = 1.043\n",
      "2018-07-25T21:12:16.987763: Epoch   1 Batch 1155/3125   train_loss = 1.124\n",
      "2018-07-25T21:12:18.104827: Epoch   1 Batch 1175/3125   train_loss = 1.139\n",
      "2018-07-25T21:12:19.283894: Epoch   1 Batch 1195/3125   train_loss = 1.258\n",
      "2018-07-25T21:12:20.412959: Epoch   1 Batch 1215/3125   train_loss = 0.967\n",
      "2018-07-25T21:12:21.612028: Epoch   1 Batch 1235/3125   train_loss = 1.110\n",
      "2018-07-25T21:12:22.826097: Epoch   1 Batch 1255/3125   train_loss = 0.974\n",
      "2018-07-25T21:12:24.023165: Epoch   1 Batch 1275/3125   train_loss = 1.031\n",
      "2018-07-25T21:12:25.246235: Epoch   1 Batch 1295/3125   train_loss = 1.058\n",
      "2018-07-25T21:12:26.359299: Epoch   1 Batch 1315/3125   train_loss = 1.176\n",
      "2018-07-25T21:12:27.539367: Epoch   1 Batch 1335/3125   train_loss = 1.020\n",
      "2018-07-25T21:12:28.646430: Epoch   1 Batch 1355/3125   train_loss = 1.045\n",
      "2018-07-25T21:12:29.847499: Epoch   1 Batch 1375/3125   train_loss = 1.108\n",
      "2018-07-25T21:12:31.050567: Epoch   1 Batch 1395/3125   train_loss = 1.048\n",
      "2018-07-25T21:12:32.212634: Epoch   1 Batch 1415/3125   train_loss = 1.093\n",
      "2018-07-25T21:12:33.409702: Epoch   1 Batch 1435/3125   train_loss = 1.200\n",
      "2018-07-25T21:12:34.588770: Epoch   1 Batch 1455/3125   train_loss = 1.232\n",
      "2018-07-25T21:12:35.824840: Epoch   1 Batch 1475/3125   train_loss = 1.131\n",
      "2018-07-25T21:12:37.097913: Epoch   1 Batch 1495/3125   train_loss = 1.034\n",
      "2018-07-25T21:12:38.263980: Epoch   1 Batch 1515/3125   train_loss = 0.964\n",
      "2018-07-25T21:12:39.418046: Epoch   1 Batch 1535/3125   train_loss = 0.945\n",
      "2018-07-25T21:12:40.608114: Epoch   1 Batch 1555/3125   train_loss = 1.067\n",
      "2018-07-25T21:12:41.725178: Epoch   1 Batch 1575/3125   train_loss = 1.024\n",
      "2018-07-25T21:12:42.857243: Epoch   1 Batch 1595/3125   train_loss = 1.116\n",
      "2018-07-25T21:12:44.042310: Epoch   1 Batch 1615/3125   train_loss = 1.063\n",
      "2018-07-25T21:12:45.232379: Epoch   1 Batch 1635/3125   train_loss = 1.117\n",
      "2018-07-25T21:12:46.395445: Epoch   1 Batch 1655/3125   train_loss = 1.119\n",
      "2018-07-25T21:12:47.502508: Epoch   1 Batch 1675/3125   train_loss = 1.008\n",
      "2018-07-25T21:12:48.627573: Epoch   1 Batch 1695/3125   train_loss = 1.053\n",
      "2018-07-25T21:12:49.829641: Epoch   1 Batch 1715/3125   train_loss = 1.023\n",
      "2018-07-25T21:12:51.025710: Epoch   1 Batch 1735/3125   train_loss = 1.200\n",
      "2018-07-25T21:12:52.257780: Epoch   1 Batch 1755/3125   train_loss = 0.994\n",
      "2018-07-25T21:12:53.406846: Epoch   1 Batch 1775/3125   train_loss = 1.078\n",
      "2018-07-25T21:12:54.595914: Epoch   1 Batch 1795/3125   train_loss = 1.062\n",
      "2018-07-25T21:12:55.800983: Epoch   1 Batch 1815/3125   train_loss = 1.043\n",
      "2018-07-25T21:12:56.946049: Epoch   1 Batch 1835/3125   train_loss = 1.165\n",
      "2018-07-25T21:12:58.115115: Epoch   1 Batch 1855/3125   train_loss = 0.956\n",
      "2018-07-25T21:12:59.260181: Epoch   1 Batch 1875/3125   train_loss = 1.060\n",
      "2018-07-25T21:13:00.424247: Epoch   1 Batch 1895/3125   train_loss = 1.002\n",
      "2018-07-25T21:13:01.625316: Epoch   1 Batch 1915/3125   train_loss = 0.873\n",
      "2018-07-25T21:13:02.796383: Epoch   1 Batch 1935/3125   train_loss = 0.995\n",
      "2018-07-25T21:13:03.987451: Epoch   1 Batch 1955/3125   train_loss = 0.997\n",
      "2018-07-25T21:13:05.145517: Epoch   1 Batch 1975/3125   train_loss = 1.060\n",
      "2018-07-25T21:13:06.210578: Epoch   1 Batch 1995/3125   train_loss = 1.185\n",
      "2018-07-25T21:13:07.352644: Epoch   1 Batch 2015/3125   train_loss = 1.115\n",
      "2018-07-25T21:13:08.484708: Epoch   1 Batch 2035/3125   train_loss = 1.070\n",
      "2018-07-25T21:13:09.600772: Epoch   1 Batch 2055/3125   train_loss = 0.958\n",
      "2018-07-25T21:13:10.733837: Epoch   1 Batch 2075/3125   train_loss = 1.174\n",
      "2018-07-25T21:13:11.878903: Epoch   1 Batch 2095/3125   train_loss = 0.960\n",
      "2018-07-25T21:13:12.983966: Epoch   1 Batch 2115/3125   train_loss = 1.068\n",
      "2018-07-25T21:13:14.169034: Epoch   1 Batch 2135/3125   train_loss = 1.022\n",
      "2018-07-25T21:13:15.304099: Epoch   1 Batch 2155/3125   train_loss = 1.093\n",
      "2018-07-25T21:13:16.430163: Epoch   1 Batch 2175/3125   train_loss = 1.017\n",
      "2018-07-25T21:13:17.483223: Epoch   1 Batch 2195/3125   train_loss = 1.099\n",
      "2018-07-25T21:13:18.686292: Epoch   1 Batch 2215/3125   train_loss = 1.011\n",
      "2018-07-25T21:13:19.830357: Epoch   1 Batch 2235/3125   train_loss = 1.141\n",
      "2018-07-25T21:13:21.023426: Epoch   1 Batch 2255/3125   train_loss = 1.077\n",
      "2018-07-25T21:13:22.183492: Epoch   1 Batch 2275/3125   train_loss = 0.882\n",
      "2018-07-25T21:13:23.332558: Epoch   1 Batch 2295/3125   train_loss = 1.170\n",
      "2018-07-25T21:13:24.485624: Epoch   1 Batch 2315/3125   train_loss = 1.127\n",
      "2018-07-25T21:13:25.696693: Epoch   1 Batch 2335/3125   train_loss = 1.056\n",
      "2018-07-25T21:13:26.873760: Epoch   1 Batch 2355/3125   train_loss = 1.045\n",
      "2018-07-25T21:13:28.016826: Epoch   1 Batch 2375/3125   train_loss = 1.278\n",
      "2018-07-25T21:13:29.177892: Epoch   1 Batch 2395/3125   train_loss = 1.048\n",
      "2018-07-25T21:13:30.359960: Epoch   1 Batch 2415/3125   train_loss = 1.113\n",
      "2018-07-25T21:13:31.553028: Epoch   1 Batch 2435/3125   train_loss = 0.978\n",
      "2018-07-25T21:13:32.701094: Epoch   1 Batch 2455/3125   train_loss = 1.059\n",
      "2018-07-25T21:13:33.875161: Epoch   1 Batch 2475/3125   train_loss = 1.003\n",
      "2018-07-25T21:13:34.996225: Epoch   1 Batch 2495/3125   train_loss = 0.971\n",
      "2018-07-25T21:13:36.178292: Epoch   1 Batch 2515/3125   train_loss = 1.031\n",
      "2018-07-25T21:13:37.435364: Epoch   1 Batch 2535/3125   train_loss = 1.109\n",
      "2018-07-25T21:13:38.606431: Epoch   1 Batch 2555/3125   train_loss = 0.908\n",
      "2018-07-25T21:13:39.793499: Epoch   1 Batch 2575/3125   train_loss = 0.899\n",
      "2018-07-25T21:13:41.002568: Epoch   1 Batch 2595/3125   train_loss = 0.958\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-07-25T21:13:42.205637: Epoch   1 Batch 2615/3125   train_loss = 1.113\n",
      "2018-07-25T21:13:43.370704: Epoch   1 Batch 2635/3125   train_loss = 0.968\n",
      "2018-07-25T21:13:44.487768: Epoch   1 Batch 2655/3125   train_loss = 1.092\n",
      "2018-07-25T21:13:45.698837: Epoch   1 Batch 2675/3125   train_loss = 0.976\n",
      "2018-07-25T21:13:46.847903: Epoch   1 Batch 2695/3125   train_loss = 1.043\n",
      "2018-07-25T21:13:48.030970: Epoch   1 Batch 2715/3125   train_loss = 0.996\n",
      "2018-07-25T21:13:49.207038: Epoch   1 Batch 2735/3125   train_loss = 0.918\n",
      "2018-07-25T21:13:50.388105: Epoch   1 Batch 2755/3125   train_loss = 1.090\n",
      "2018-07-25T21:13:51.561172: Epoch   1 Batch 2775/3125   train_loss = 1.105\n",
      "2018-07-25T21:13:52.681236: Epoch   1 Batch 2795/3125   train_loss = 1.079\n",
      "2018-07-25T21:13:53.852303: Epoch   1 Batch 2815/3125   train_loss = 0.974\n",
      "2018-07-25T21:13:55.212381: Epoch   1 Batch 2835/3125   train_loss = 1.078\n",
      "2018-07-25T21:13:56.368447: Epoch   1 Batch 2855/3125   train_loss = 1.083\n",
      "2018-07-25T21:13:57.535514: Epoch   1 Batch 2875/3125   train_loss = 0.991\n",
      "2018-07-25T21:13:58.697581: Epoch   1 Batch 2895/3125   train_loss = 1.009\n",
      "2018-07-25T21:13:59.850646: Epoch   1 Batch 2915/3125   train_loss = 0.936\n",
      "2018-07-25T21:14:00.995712: Epoch   1 Batch 2935/3125   train_loss = 1.072\n",
      "2018-07-25T21:14:02.187780: Epoch   1 Batch 2955/3125   train_loss = 1.115\n",
      "2018-07-25T21:14:03.417850: Epoch   1 Batch 2975/3125   train_loss = 0.979\n",
      "2018-07-25T21:14:04.580917: Epoch   1 Batch 2995/3125   train_loss = 0.923\n",
      "2018-07-25T21:14:05.824988: Epoch   1 Batch 3015/3125   train_loss = 0.986\n",
      "2018-07-25T21:14:07.024057: Epoch   1 Batch 3035/3125   train_loss = 1.021\n",
      "2018-07-25T21:14:08.165122: Epoch   1 Batch 3055/3125   train_loss = 1.078\n",
      "2018-07-25T21:14:09.336189: Epoch   1 Batch 3075/3125   train_loss = 1.058\n",
      "2018-07-25T21:14:10.460253: Epoch   1 Batch 3095/3125   train_loss = 1.000\n",
      "2018-07-25T21:14:11.654322: Epoch   1 Batch 3115/3125   train_loss = 0.837\n",
      "2018-07-25T21:14:12.573374: Epoch   1 Batch   19/781   test_loss = 1.074\n",
      "2018-07-25T21:14:12.898393: Epoch   1 Batch   39/781   test_loss = 0.816\n",
      "2018-07-25T21:14:13.196410: Epoch   1 Batch   59/781   test_loss = 0.918\n",
      "2018-07-25T21:14:13.500427: Epoch   1 Batch   79/781   test_loss = 1.012\n",
      "2018-07-25T21:14:13.800444: Epoch   1 Batch   99/781   test_loss = 0.971\n",
      "2018-07-25T21:14:14.098461: Epoch   1 Batch  119/781   test_loss = 0.994\n",
      "2018-07-25T21:14:14.397478: Epoch   1 Batch  139/781   test_loss = 1.065\n",
      "2018-07-25T21:14:14.700496: Epoch   1 Batch  159/781   test_loss = 1.049\n",
      "2018-07-25T21:14:15.005513: Epoch   1 Batch  179/781   test_loss = 0.915\n",
      "2018-07-25T21:14:15.327532: Epoch   1 Batch  199/781   test_loss = 0.924\n",
      "2018-07-25T21:14:15.664551: Epoch   1 Batch  219/781   test_loss = 1.038\n",
      "2018-07-25T21:14:16.002570: Epoch   1 Batch  239/781   test_loss = 1.307\n",
      "2018-07-25T21:14:16.334589: Epoch   1 Batch  259/781   test_loss = 0.934\n",
      "2018-07-25T21:14:16.669608: Epoch   1 Batch  279/781   test_loss = 1.154\n",
      "2018-07-25T21:14:16.993627: Epoch   1 Batch  299/781   test_loss = 1.192\n",
      "2018-07-25T21:14:17.315645: Epoch   1 Batch  319/781   test_loss = 0.951\n",
      "2018-07-25T21:14:17.623663: Epoch   1 Batch  339/781   test_loss = 0.967\n",
      "2018-07-25T21:14:17.929681: Epoch   1 Batch  359/781   test_loss = 0.929\n",
      "2018-07-25T21:14:18.258699: Epoch   1 Batch  379/781   test_loss = 0.988\n",
      "2018-07-25T21:14:18.577718: Epoch   1 Batch  399/781   test_loss = 0.876\n",
      "2018-07-25T21:14:18.881735: Epoch   1 Batch  419/781   test_loss = 0.989\n",
      "2018-07-25T21:14:19.201753: Epoch   1 Batch  439/781   test_loss = 1.050\n",
      "2018-07-25T21:14:19.504771: Epoch   1 Batch  459/781   test_loss = 1.094\n",
      "2018-07-25T21:14:19.815788: Epoch   1 Batch  479/781   test_loss = 1.046\n",
      "2018-07-25T21:14:20.128806: Epoch   1 Batch  499/781   test_loss = 0.934\n",
      "2018-07-25T21:14:20.442824: Epoch   1 Batch  519/781   test_loss = 1.156\n",
      "2018-07-25T21:14:20.756842: Epoch   1 Batch  539/781   test_loss = 0.854\n",
      "2018-07-25T21:14:21.068860: Epoch   1 Batch  559/781   test_loss = 1.118\n",
      "2018-07-25T21:14:21.374878: Epoch   1 Batch  579/781   test_loss = 1.036\n",
      "2018-07-25T21:14:21.674895: Epoch   1 Batch  599/781   test_loss = 0.911\n",
      "2018-07-25T21:14:21.976912: Epoch   1 Batch  619/781   test_loss = 1.228\n",
      "2018-07-25T21:14:22.280929: Epoch   1 Batch  639/781   test_loss = 0.930\n",
      "2018-07-25T21:14:22.598948: Epoch   1 Batch  659/781   test_loss = 1.156\n",
      "2018-07-25T21:14:22.928966: Epoch   1 Batch  679/781   test_loss = 1.127\n",
      "2018-07-25T21:14:23.239984: Epoch   1 Batch  699/781   test_loss = 0.864\n",
      "2018-07-25T21:14:23.544002: Epoch   1 Batch  719/781   test_loss = 0.989\n",
      "2018-07-25T21:14:23.848019: Epoch   1 Batch  739/781   test_loss = 0.981\n",
      "2018-07-25T21:14:24.153036: Epoch   1 Batch  759/781   test_loss = 0.911\n",
      "2018-07-25T21:14:24.451054: Epoch   1 Batch  779/781   test_loss = 0.783\n",
      "2018-07-25T21:14:25.690124: Epoch   2 Batch   10/3125   train_loss = 0.933\n",
      "2018-07-25T21:14:26.850191: Epoch   2 Batch   30/3125   train_loss = 1.045\n",
      "2018-07-25T21:14:28.072261: Epoch   2 Batch   50/3125   train_loss = 1.096\n",
      "2018-07-25T21:14:29.243328: Epoch   2 Batch   70/3125   train_loss = 1.057\n",
      "2018-07-25T21:14:30.392393: Epoch   2 Batch   90/3125   train_loss = 1.014\n",
      "2018-07-25T21:14:31.545459: Epoch   2 Batch  110/3125   train_loss = 0.911\n",
      "2018-07-25T21:14:32.701525: Epoch   2 Batch  130/3125   train_loss = 0.958\n",
      "2018-07-25T21:14:33.897594: Epoch   2 Batch  150/3125   train_loss = 1.077\n",
      "2018-07-25T21:14:35.069661: Epoch   2 Batch  170/3125   train_loss = 0.984\n",
      "2018-07-25T21:14:36.265729: Epoch   2 Batch  190/3125   train_loss = 1.036\n",
      "2018-07-25T21:14:37.437796: Epoch   2 Batch  210/3125   train_loss = 0.934\n",
      "2018-07-25T21:14:38.636865: Epoch   2 Batch  230/3125   train_loss = 1.006\n",
      "2018-07-25T21:14:39.832933: Epoch   2 Batch  250/3125   train_loss = 0.933\n",
      "2018-07-25T21:14:41.035002: Epoch   2 Batch  270/3125   train_loss = 0.797\n",
      "2018-07-25T21:14:42.142065: Epoch   2 Batch  290/3125   train_loss = 1.102\n",
      "2018-07-25T21:14:43.281131: Epoch   2 Batch  310/3125   train_loss = 0.932\n",
      "2018-07-25T21:14:44.426196: Epoch   2 Batch  330/3125   train_loss = 1.026\n",
      "2018-07-25T21:14:45.668267: Epoch   2 Batch  350/3125   train_loss = 0.953\n",
      "2018-07-25T21:14:46.796332: Epoch   2 Batch  370/3125   train_loss = 1.125\n",
      "2018-07-25T21:14:47.895394: Epoch   2 Batch  390/3125   train_loss = 1.174\n",
      "2018-07-25T21:14:49.053461: Epoch   2 Batch  410/3125   train_loss = 0.893\n",
      "2018-07-25T21:14:50.216527: Epoch   2 Batch  430/3125   train_loss = 1.155\n",
      "2018-07-25T21:14:51.341592: Epoch   2 Batch  450/3125   train_loss = 0.958\n",
      "2018-07-25T21:14:52.453655: Epoch   2 Batch  470/3125   train_loss = 0.947\n",
      "2018-07-25T21:14:53.638723: Epoch   2 Batch  490/3125   train_loss = 1.051\n",
      "2018-07-25T21:14:54.819791: Epoch   2 Batch  510/3125   train_loss = 1.114\n",
      "2018-07-25T21:14:55.936854: Epoch   2 Batch  530/3125   train_loss = 0.959\n",
      "2018-07-25T21:14:57.056918: Epoch   2 Batch  550/3125   train_loss = 1.000\n",
      "2018-07-25T21:14:58.168982: Epoch   2 Batch  570/3125   train_loss = 1.125\n",
      "2018-07-25T21:14:59.363050: Epoch   2 Batch  590/3125   train_loss = 1.073\n",
      "2018-07-25T21:15:00.534117: Epoch   2 Batch  610/3125   train_loss = 0.928\n",
      "2018-07-25T21:15:01.709185: Epoch   2 Batch  630/3125   train_loss = 1.038\n",
      "2018-07-25T21:15:02.863251: Epoch   2 Batch  650/3125   train_loss = 1.025\n",
      "2018-07-25T21:15:03.955313: Epoch   2 Batch  670/3125   train_loss = 0.931\n",
      "2018-07-25T21:15:05.105379: Epoch   2 Batch  690/3125   train_loss = 0.915\n",
      "2018-07-25T21:15:06.295447: Epoch   2 Batch  710/3125   train_loss = 0.870\n",
      "2018-07-25T21:15:07.438512: Epoch   2 Batch  730/3125   train_loss = 0.775\n",
      "2018-07-25T21:15:08.577577: Epoch   2 Batch  750/3125   train_loss = 0.961\n",
      "2018-07-25T21:15:09.820649: Epoch   2 Batch  770/3125   train_loss = 0.889\n",
      "2018-07-25T21:15:10.958714: Epoch   2 Batch  790/3125   train_loss = 0.942\n",
      "2018-07-25T21:15:12.099779: Epoch   2 Batch  810/3125   train_loss = 0.903\n",
      "2018-07-25T21:15:13.229844: Epoch   2 Batch  830/3125   train_loss = 0.804\n",
      "2018-07-25T21:15:14.333907: Epoch   2 Batch  850/3125   train_loss = 1.057\n",
      "2018-07-25T21:15:15.358965: Epoch   2 Batch  870/3125   train_loss = 0.915\n",
      "2018-07-25T21:15:16.481029: Epoch   2 Batch  890/3125   train_loss = 0.955\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-07-25T21:15:17.564091: Epoch   2 Batch  910/3125   train_loss = 1.020\n",
      "2018-07-25T21:15:18.714157: Epoch   2 Batch  930/3125   train_loss = 1.014\n",
      "2018-07-25T21:15:19.860223: Epoch   2 Batch  950/3125   train_loss = 0.997\n",
      "2018-07-25T21:15:21.041290: Epoch   2 Batch  970/3125   train_loss = 1.015\n",
      "2018-07-25T21:15:22.167355: Epoch   2 Batch  990/3125   train_loss = 0.865\n",
      "2018-07-25T21:15:23.260417: Epoch   2 Batch 1010/3125   train_loss = 1.110\n",
      "2018-07-25T21:15:24.393482: Epoch   2 Batch 1030/3125   train_loss = 0.949\n",
      "2018-07-25T21:15:25.557549: Epoch   2 Batch 1050/3125   train_loss = 0.997\n",
      "2018-07-25T21:15:26.751617: Epoch   2 Batch 1070/3125   train_loss = 0.979\n",
      "2018-07-25T21:15:27.965686: Epoch   2 Batch 1090/3125   train_loss = 1.001\n",
      "2018-07-25T21:15:29.163755: Epoch   2 Batch 1110/3125   train_loss = 1.064\n",
      "2018-07-25T21:15:30.346823: Epoch   2 Batch 1130/3125   train_loss = 0.892\n",
      "2018-07-25T21:15:31.532890: Epoch   2 Batch 1150/3125   train_loss = 0.950\n",
      "2018-07-25T21:15:32.632953: Epoch   2 Batch 1170/3125   train_loss = 0.948\n",
      "2018-07-25T21:15:33.780019: Epoch   2 Batch 1190/3125   train_loss = 1.042\n",
      "2018-07-25T21:15:34.944085: Epoch   2 Batch 1210/3125   train_loss = 0.884\n",
      "2018-07-25T21:15:36.148154: Epoch   2 Batch 1230/3125   train_loss = 0.903\n",
      "2018-07-25T21:15:37.396226: Epoch   2 Batch 1250/3125   train_loss = 1.046\n",
      "2018-07-25T21:15:38.610295: Epoch   2 Batch 1270/3125   train_loss = 0.986\n",
      "2018-07-25T21:15:39.760361: Epoch   2 Batch 1290/3125   train_loss = 0.958\n",
      "2018-07-25T21:15:40.921427: Epoch   2 Batch 1310/3125   train_loss = 0.955\n",
      "2018-07-25T21:15:42.121496: Epoch   2 Batch 1330/3125   train_loss = 1.040\n",
      "2018-07-25T21:15:43.268562: Epoch   2 Batch 1350/3125   train_loss = 0.933\n",
      "2018-07-25T21:15:44.450629: Epoch   2 Batch 1370/3125   train_loss = 0.850\n",
      "2018-07-25T21:15:45.579694: Epoch   2 Batch 1390/3125   train_loss = 0.998\n",
      "2018-07-25T21:15:46.739760: Epoch   2 Batch 1410/3125   train_loss = 0.926\n",
      "2018-07-25T21:15:47.949829: Epoch   2 Batch 1430/3125   train_loss = 1.006\n",
      "2018-07-25T21:15:49.129897: Epoch   2 Batch 1450/3125   train_loss = 1.031\n",
      "2018-07-25T21:15:50.300964: Epoch   2 Batch 1470/3125   train_loss = 0.978\n",
      "2018-07-25T21:15:51.436029: Epoch   2 Batch 1490/3125   train_loss = 1.017\n",
      "2018-07-25T21:15:52.598095: Epoch   2 Batch 1510/3125   train_loss = 0.961\n",
      "2018-07-25T21:15:53.769162: Epoch   2 Batch 1530/3125   train_loss = 1.074\n",
      "2018-07-25T21:15:55.130240: Epoch   2 Batch 1550/3125   train_loss = 0.849\n",
      "2018-07-25T21:15:56.318308: Epoch   2 Batch 1570/3125   train_loss = 0.893\n",
      "2018-07-25T21:15:57.505376: Epoch   2 Batch 1590/3125   train_loss = 0.988\n",
      "2018-07-25T21:15:58.705445: Epoch   2 Batch 1610/3125   train_loss = 0.997\n",
      "2018-07-25T21:15:59.881512: Epoch   2 Batch 1630/3125   train_loss = 1.041\n",
      "2018-07-25T21:16:01.024577: Epoch   2 Batch 1650/3125   train_loss = 0.853\n",
      "2018-07-25T21:16:02.268648: Epoch   2 Batch 1670/3125   train_loss = 0.836\n",
      "2018-07-25T21:16:03.489718: Epoch   2 Batch 1690/3125   train_loss = 1.008\n",
      "2018-07-25T21:16:04.701788: Epoch   2 Batch 1710/3125   train_loss = 0.915\n",
      "2018-07-25T21:16:05.894856: Epoch   2 Batch 1730/3125   train_loss = 0.927\n",
      "2018-07-25T21:16:07.126926: Epoch   2 Batch 1750/3125   train_loss = 0.828\n",
      "2018-07-25T21:16:08.405999: Epoch   2 Batch 1770/3125   train_loss = 1.133\n",
      "2018-07-25T21:16:09.629069: Epoch   2 Batch 1790/3125   train_loss = 0.968\n",
      "2018-07-25T21:16:10.806137: Epoch   2 Batch 1810/3125   train_loss = 1.018\n",
      "2018-07-25T21:16:12.059208: Epoch   2 Batch 1830/3125   train_loss = 1.061\n",
      "2018-07-25T21:16:13.235276: Epoch   2 Batch 1850/3125   train_loss = 0.961\n",
      "2018-07-25T21:16:14.434344: Epoch   2 Batch 1870/3125   train_loss = 1.019\n",
      "2018-07-25T21:16:15.626412: Epoch   2 Batch 1890/3125   train_loss = 0.822\n",
      "2018-07-25T21:16:16.836482: Epoch   2 Batch 1910/3125   train_loss = 0.916\n",
      "2018-07-25T21:16:17.925544: Epoch   2 Batch 1930/3125   train_loss = 1.069\n",
      "2018-07-25T21:16:18.991605: Epoch   2 Batch 1950/3125   train_loss = 0.843\n",
      "2018-07-25T21:16:20.023664: Epoch   2 Batch 1970/3125   train_loss = 1.012\n",
      "2018-07-25T21:16:21.148728: Epoch   2 Batch 1990/3125   train_loss = 0.836\n",
      "2018-07-25T21:16:22.364798: Epoch   2 Batch 2010/3125   train_loss = 0.822\n",
      "2018-07-25T21:16:23.581867: Epoch   2 Batch 2030/3125   train_loss = 0.901\n",
      "2018-07-25T21:16:24.742934: Epoch   2 Batch 2050/3125   train_loss = 1.030\n",
      "2018-07-25T21:16:25.948003: Epoch   2 Batch 2070/3125   train_loss = 0.959\n",
      "2018-07-25T21:16:27.146071: Epoch   2 Batch 2090/3125   train_loss = 0.865\n",
      "2018-07-25T21:16:28.336139: Epoch   2 Batch 2110/3125   train_loss = 1.035\n",
      "2018-07-25T21:16:29.514207: Epoch   2 Batch 2130/3125   train_loss = 0.974\n",
      "2018-07-25T21:16:30.680273: Epoch   2 Batch 2150/3125   train_loss = 0.980\n",
      "2018-07-25T21:16:31.794337: Epoch   2 Batch 2170/3125   train_loss = 0.890\n",
      "2018-07-25T21:16:32.983405: Epoch   2 Batch 2190/3125   train_loss = 0.939\n",
      "2018-07-25T21:16:34.091469: Epoch   2 Batch 2210/3125   train_loss = 1.007\n",
      "2018-07-25T21:16:35.307538: Epoch   2 Batch 2230/3125   train_loss = 0.882\n",
      "2018-07-25T21:16:36.460604: Epoch   2 Batch 2250/3125   train_loss = 1.016\n",
      "2018-07-25T21:16:37.632671: Epoch   2 Batch 2270/3125   train_loss = 0.951\n",
      "2018-07-25T21:16:38.714733: Epoch   2 Batch 2290/3125   train_loss = 0.826\n",
      "2018-07-25T21:16:39.825797: Epoch   2 Batch 2310/3125   train_loss = 0.873\n",
      "2018-07-25T21:16:40.927860: Epoch   2 Batch 2330/3125   train_loss = 1.059\n",
      "2018-07-25T21:16:42.027922: Epoch   2 Batch 2350/3125   train_loss = 1.075\n",
      "2018-07-25T21:16:43.096984: Epoch   2 Batch 2370/3125   train_loss = 0.939\n",
      "2018-07-25T21:16:44.216048: Epoch   2 Batch 2390/3125   train_loss = 1.006\n",
      "2018-07-25T21:16:45.308110: Epoch   2 Batch 2410/3125   train_loss = 1.058\n",
      "2018-07-25T21:16:46.411173: Epoch   2 Batch 2430/3125   train_loss = 0.936\n",
      "2018-07-25T21:16:47.580240: Epoch   2 Batch 2450/3125   train_loss = 0.976\n",
      "2018-07-25T21:16:48.722305: Epoch   2 Batch 2470/3125   train_loss = 1.034\n",
      "2018-07-25T21:16:49.916374: Epoch   2 Batch 2490/3125   train_loss = 0.989\n",
      "2018-07-25T21:16:51.033438: Epoch   2 Batch 2510/3125   train_loss = 1.031\n",
      "2018-07-25T21:16:52.236506: Epoch   2 Batch 2530/3125   train_loss = 0.860\n",
      "2018-07-25T21:16:53.423574: Epoch   2 Batch 2550/3125   train_loss = 0.964\n",
      "2018-07-25T21:16:54.581640: Epoch   2 Batch 2570/3125   train_loss = 1.026\n",
      "2018-07-25T21:16:55.802710: Epoch   2 Batch 2590/3125   train_loss = 1.008\n",
      "2018-07-25T21:16:56.939775: Epoch   2 Batch 2610/3125   train_loss = 1.059\n",
      "2018-07-25T21:16:58.023837: Epoch   2 Batch 2630/3125   train_loss = 0.684\n",
      "2018-07-25T21:16:59.186904: Epoch   2 Batch 2650/3125   train_loss = 0.983\n",
      "2018-07-25T21:17:00.297967: Epoch   2 Batch 2670/3125   train_loss = 0.945\n",
      "2018-07-25T21:17:01.447033: Epoch   2 Batch 2690/3125   train_loss = 1.032\n",
      "2018-07-25T21:17:02.522095: Epoch   2 Batch 2710/3125   train_loss = 0.808\n",
      "2018-07-25T21:17:03.636158: Epoch   2 Batch 2730/3125   train_loss = 1.118\n",
      "2018-07-25T21:17:04.731221: Epoch   2 Batch 2750/3125   train_loss = 0.978\n",
      "2018-07-25T21:17:05.844285: Epoch   2 Batch 2770/3125   train_loss = 0.889\n",
      "2018-07-25T21:17:06.944348: Epoch   2 Batch 2790/3125   train_loss = 0.946\n",
      "2018-07-25T21:17:08.016409: Epoch   2 Batch 2810/3125   train_loss = 0.944\n",
      "2018-07-25T21:17:09.148474: Epoch   2 Batch 2830/3125   train_loss = 0.881\n",
      "2018-07-25T21:17:10.222535: Epoch   2 Batch 2850/3125   train_loss = 0.974\n",
      "2018-07-25T21:17:11.369601: Epoch   2 Batch 2870/3125   train_loss = 0.814\n",
      "2018-07-25T21:17:12.476664: Epoch   2 Batch 2890/3125   train_loss = 0.812\n",
      "2018-07-25T21:17:13.588728: Epoch   2 Batch 2910/3125   train_loss = 0.937\n",
      "2018-07-25T21:17:14.696791: Epoch   2 Batch 2930/3125   train_loss = 0.868\n",
      "2018-07-25T21:17:15.860858: Epoch   2 Batch 2950/3125   train_loss = 1.120\n",
      "2018-07-25T21:17:16.899917: Epoch   2 Batch 2970/3125   train_loss = 0.963\n",
      "2018-07-25T21:17:17.964978: Epoch   2 Batch 2990/3125   train_loss = 0.888\n",
      "2018-07-25T21:17:19.039039: Epoch   2 Batch 3010/3125   train_loss = 0.960\n",
      "2018-07-25T21:17:20.170104: Epoch   2 Batch 3030/3125   train_loss = 0.940\n",
      "2018-07-25T21:17:21.279168: Epoch   2 Batch 3050/3125   train_loss = 0.969\n",
      "2018-07-25T21:17:22.408232: Epoch   2 Batch 3070/3125   train_loss = 0.908\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-07-25T21:17:23.551297: Epoch   2 Batch 3090/3125   train_loss = 0.817\n",
      "2018-07-25T21:17:24.710364: Epoch   2 Batch 3110/3125   train_loss = 0.852\n",
      "2018-07-25T21:17:25.877431: Epoch   2 Batch   18/781   test_loss = 0.772\n",
      "2018-07-25T21:17:26.201449: Epoch   2 Batch   38/781   test_loss = 0.932\n",
      "2018-07-25T21:17:26.506466: Epoch   2 Batch   58/781   test_loss = 0.860\n",
      "2018-07-25T21:17:39.941235: Epoch   2 Batch   78/781   test_loss = 0.868\n",
      "2018-07-25T21:17:40.290255: Epoch   2 Batch   98/781   test_loss = 0.955\n",
      "2018-07-25T21:17:40.601273: Epoch   2 Batch  118/781   test_loss = 0.797\n",
      "2018-07-25T21:17:40.905290: Epoch   2 Batch  138/781   test_loss = 0.995\n",
      "2018-07-25T21:17:41.192306: Epoch   2 Batch  158/781   test_loss = 0.856\n",
      "2018-07-25T21:17:41.496324: Epoch   2 Batch  178/781   test_loss = 0.836\n",
      "2018-07-25T21:17:41.785340: Epoch   2 Batch  198/781   test_loss = 0.897\n",
      "2018-07-25T21:17:42.086358: Epoch   2 Batch  218/781   test_loss = 1.067\n",
      "2018-07-25T21:17:42.386375: Epoch   2 Batch  238/781   test_loss = 0.943\n",
      "2018-07-25T21:17:42.681392: Epoch   2 Batch  258/781   test_loss = 0.996\n",
      "2018-07-25T21:17:42.974408: Epoch   2 Batch  278/781   test_loss = 1.050\n",
      "2018-07-25T21:17:43.340429: Epoch   2 Batch  298/781   test_loss = 0.894\n",
      "2018-07-25T21:17:43.633446: Epoch   2 Batch  318/781   test_loss = 0.885\n",
      "2018-07-25T21:17:43.933463: Epoch   2 Batch  338/781   test_loss = 0.917\n",
      "2018-07-25T21:17:44.236481: Epoch   2 Batch  358/781   test_loss = 0.909\n",
      "2018-07-25T21:17:44.542498: Epoch   2 Batch  378/781   test_loss = 0.848\n",
      "2018-07-25T21:17:44.851516: Epoch   2 Batch  398/781   test_loss = 0.848\n",
      "2018-07-25T21:17:45.157533: Epoch   2 Batch  418/781   test_loss = 0.997\n",
      "2018-07-25T21:17:45.468551: Epoch   2 Batch  438/781   test_loss = 1.051\n",
      "2018-07-25T21:17:45.771568: Epoch   2 Batch  458/781   test_loss = 0.929\n",
      "2018-07-25T21:17:46.095587: Epoch   2 Batch  478/781   test_loss = 0.987\n",
      "2018-07-25T21:17:46.430606: Epoch   2 Batch  498/781   test_loss = 0.824\n",
      "2018-07-25T21:17:46.722623: Epoch   2 Batch  518/781   test_loss = 0.930\n",
      "2018-07-25T21:17:47.032641: Epoch   2 Batch  538/781   test_loss = 0.812\n",
      "2018-07-25T21:17:47.341658: Epoch   2 Batch  558/781   test_loss = 0.912\n",
      "2018-07-25T21:17:47.660676: Epoch   2 Batch  578/781   test_loss = 0.964\n",
      "2018-07-25T21:17:48.009696: Epoch   2 Batch  598/781   test_loss = 1.047\n",
      "2018-07-25T21:17:48.332715: Epoch   2 Batch  618/781   test_loss = 0.866\n",
      "2018-07-25T21:17:48.656733: Epoch   2 Batch  638/781   test_loss = 0.878\n",
      "2018-07-25T21:17:48.972751: Epoch   2 Batch  658/781   test_loss = 1.080\n",
      "2018-07-25T21:17:49.743796: Epoch   2 Batch  678/781   test_loss = 0.857\n",
      "2018-07-25T21:17:50.056813: Epoch   2 Batch  698/781   test_loss = 0.957\n",
      "2018-07-25T21:17:50.366831: Epoch   2 Batch  718/781   test_loss = 1.000\n",
      "2018-07-25T21:17:50.684849: Epoch   2 Batch  738/781   test_loss = 0.858\n",
      "2018-07-25T21:17:50.998867: Epoch   2 Batch  758/781   test_loss = 0.972\n",
      "2018-07-25T21:17:51.309885: Epoch   2 Batch  778/781   test_loss = 0.881\n",
      "2018-07-25T21:17:52.678963: Epoch   3 Batch    5/3125   train_loss = 0.911\n",
      "2018-07-25T21:17:54.454065: Epoch   3 Batch   25/3125   train_loss = 1.013\n",
      "2018-07-25T21:17:55.709137: Epoch   3 Batch   45/3125   train_loss = 0.812\n",
      "2018-07-25T21:17:56.884204: Epoch   3 Batch   65/3125   train_loss = 0.952\n",
      "2018-07-25T21:17:58.043270: Epoch   3 Batch   85/3125   train_loss = 0.850\n",
      "2018-07-25T21:17:59.167335: Epoch   3 Batch  105/3125   train_loss = 0.759\n",
      "2018-07-25T21:18:00.879433: Epoch   3 Batch  125/3125   train_loss = 0.897\n",
      "2018-07-25T21:18:02.065500: Epoch   3 Batch  145/3125   train_loss = 0.939\n",
      "2018-07-25T21:18:03.144562: Epoch   3 Batch  165/3125   train_loss = 0.978\n",
      "2018-07-25T21:18:04.257626: Epoch   3 Batch  185/3125   train_loss = 0.817\n",
      "2018-07-25T21:18:05.427693: Epoch   3 Batch  205/3125   train_loss = 0.872\n",
      "2018-07-25T21:18:06.575758: Epoch   3 Batch  225/3125   train_loss = 0.777\n",
      "2018-07-25T21:18:07.649820: Epoch   3 Batch  245/3125   train_loss = 1.094\n",
      "2018-07-25T21:18:08.870890: Epoch   3 Batch  265/3125   train_loss = 0.875\n",
      "2018-07-25T21:18:10.055957: Epoch   3 Batch  285/3125   train_loss = 0.935\n",
      "2018-07-25T21:18:11.130019: Epoch   3 Batch  305/3125   train_loss = 0.816\n",
      "2018-07-25T21:18:12.303086: Epoch   3 Batch  325/3125   train_loss = 0.964\n",
      "2018-07-25T21:18:13.374147: Epoch   3 Batch  345/3125   train_loss = 0.952\n",
      "2018-07-25T21:18:14.483211: Epoch   3 Batch  365/3125   train_loss = 0.903\n",
      "2018-07-25T21:18:15.600274: Epoch   3 Batch  385/3125   train_loss = 0.881\n",
      "2018-07-25T21:18:16.651335: Epoch   3 Batch  405/3125   train_loss = 0.874\n",
      "2018-07-25T21:18:17.752398: Epoch   3 Batch  425/3125   train_loss = 0.964\n",
      "2018-07-25T21:18:18.835460: Epoch   3 Batch  445/3125   train_loss = 0.947\n",
      "2018-07-25T21:18:21.607618: Epoch   3 Batch  465/3125   train_loss = 0.882\n",
      "2018-07-25T21:18:22.675679: Epoch   3 Batch  485/3125   train_loss = 1.040\n",
      "2018-07-25T21:18:23.789743: Epoch   3 Batch  505/3125   train_loss = 0.829\n",
      "2018-07-25T21:18:24.934808: Epoch   3 Batch  525/3125   train_loss = 0.993\n",
      "2018-07-25T21:18:26.004870: Epoch   3 Batch  545/3125   train_loss = 0.861\n",
      "2018-07-25T21:18:27.171936: Epoch   3 Batch  565/3125   train_loss = 1.050\n",
      "2018-07-25T21:18:30.173108: Epoch   3 Batch  585/3125   train_loss = 0.903\n",
      "2018-07-25T21:18:34.796372: Epoch   3 Batch  605/3125   train_loss = 0.882\n",
      "2018-07-25T21:18:35.922437: Epoch   3 Batch  625/3125   train_loss = 0.859\n",
      "2018-07-25T21:18:37.050501: Epoch   3 Batch  645/3125   train_loss = 0.963\n",
      "2018-07-25T21:18:38.190567: Epoch   3 Batch  665/3125   train_loss = 1.022\n",
      "2018-07-25T21:18:39.353633: Epoch   3 Batch  685/3125   train_loss = 0.900\n",
      "2018-07-25T21:18:40.634706: Epoch   3 Batch  705/3125   train_loss = 1.081\n",
      "2018-07-25T21:18:41.817774: Epoch   3 Batch  725/3125   train_loss = 0.895\n",
      "2018-07-25T21:18:42.997842: Epoch   3 Batch  745/3125   train_loss = 0.915\n",
      "2018-07-25T21:18:44.082904: Epoch   3 Batch  765/3125   train_loss = 0.873\n",
      "2018-07-25T21:18:45.208968: Epoch   3 Batch  785/3125   train_loss = 1.091\n",
      "2018-07-25T21:18:46.355034: Epoch   3 Batch  805/3125   train_loss = 0.857\n",
      "2018-07-25T21:18:47.498099: Epoch   3 Batch  825/3125   train_loss = 0.898\n",
      "2018-07-25T21:18:48.810174: Epoch   3 Batch  845/3125   train_loss = 0.911\n",
      "2018-07-25T21:18:49.934238: Epoch   3 Batch  865/3125   train_loss = 1.032\n",
      "2018-07-25T21:18:51.103305: Epoch   3 Batch  885/3125   train_loss = 0.929\n",
      "2018-07-25T21:18:52.175366: Epoch   3 Batch  905/3125   train_loss = 0.950\n",
      "2018-07-25T21:18:53.282430: Epoch   3 Batch  925/3125   train_loss = 0.872\n",
      "2018-07-25T21:18:54.394493: Epoch   3 Batch  945/3125   train_loss = 0.963\n",
      "2018-07-25T21:18:55.520558: Epoch   3 Batch  965/3125   train_loss = 0.779\n",
      "2018-07-25T21:18:56.627621: Epoch   3 Batch  985/3125   train_loss = 0.981\n",
      "2018-07-25T21:18:57.776687: Epoch   3 Batch 1005/3125   train_loss = 0.789\n",
      "2018-07-25T21:18:58.878750: Epoch   3 Batch 1025/3125   train_loss = 0.928\n",
      "2018-07-25T21:19:00.619849: Epoch   3 Batch 1045/3125   train_loss = 1.132\n",
      "2018-07-25T21:19:01.749914: Epoch   3 Batch 1065/3125   train_loss = 0.946\n",
      "2018-07-25T21:19:02.914981: Epoch   3 Batch 1085/3125   train_loss = 0.832\n",
      "2018-07-25T21:19:04.081047: Epoch   3 Batch 1105/3125   train_loss = 0.812\n",
      "2018-07-25T21:19:05.290117: Epoch   3 Batch 1125/3125   train_loss = 0.836\n",
      "2018-07-25T21:19:06.434182: Epoch   3 Batch 1145/3125   train_loss = 0.877\n",
      "2018-07-25T21:19:07.514244: Epoch   3 Batch 1165/3125   train_loss = 1.015\n",
      "2018-07-25T21:19:08.666310: Epoch   3 Batch 1185/3125   train_loss = 0.838\n",
      "2018-07-25T21:19:09.777373: Epoch   3 Batch 1205/3125   train_loss = 0.924\n",
      "2018-07-25T21:19:10.852435: Epoch   3 Batch 1225/3125   train_loss = 0.976\n",
      "2018-07-25T21:19:11.956498: Epoch   3 Batch 1245/3125   train_loss = 1.034\n",
      "2018-07-25T21:19:13.053561: Epoch   3 Batch 1265/3125   train_loss = 0.891\n",
      "2018-07-25T21:19:14.231628: Epoch   3 Batch 1285/3125   train_loss = 0.976\n",
      "2018-07-25T21:19:15.273688: Epoch   3 Batch 1305/3125   train_loss = 0.767\n",
      "2018-07-25T21:19:16.380751: Epoch   3 Batch 1325/3125   train_loss = 0.890\n",
      "2018-07-25T21:19:17.466813: Epoch   3 Batch 1345/3125   train_loss = 0.941\n",
      "2018-07-25T21:19:18.634880: Epoch   3 Batch 1365/3125   train_loss = 0.754\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-07-25T21:19:19.732943: Epoch   3 Batch 1385/3125   train_loss = 0.824\n",
      "2018-07-25T21:19:20.907010: Epoch   3 Batch 1405/3125   train_loss = 0.895\n",
      "2018-07-25T21:19:22.049075: Epoch   3 Batch 1425/3125   train_loss = 1.044\n",
      "2018-07-25T21:19:23.235143: Epoch   3 Batch 1445/3125   train_loss = 1.008\n",
      "2018-07-25T21:19:24.381209: Epoch   3 Batch 1465/3125   train_loss = 0.853\n",
      "2018-07-25T21:19:25.440269: Epoch   3 Batch 1485/3125   train_loss = 0.982\n",
      "2018-07-25T21:19:26.604336: Epoch   3 Batch 1505/3125   train_loss = 0.785\n",
      "2018-07-25T21:19:27.711399: Epoch   3 Batch 1525/3125   train_loss = 0.809\n",
      "2018-07-25T21:19:28.697455: Epoch   3 Batch 1545/3125   train_loss = 0.891\n",
      "2018-07-25T21:19:29.836521: Epoch   3 Batch 1565/3125   train_loss = 1.002\n",
      "2018-07-25T21:19:30.907582: Epoch   3 Batch 1585/3125   train_loss = 0.821\n",
      "2018-07-25T21:19:31.964642: Epoch   3 Batch 1605/3125   train_loss = 0.905\n",
      "2018-07-25T21:19:33.035704: Epoch   3 Batch 1625/3125   train_loss = 0.915\n",
      "2018-07-25T21:19:34.229772: Epoch   3 Batch 1645/3125   train_loss = 1.011\n",
      "2018-07-25T21:19:35.411839: Epoch   3 Batch 1665/3125   train_loss = 0.922\n",
      "2018-07-25T21:19:36.507902: Epoch   3 Batch 1685/3125   train_loss = 0.934\n",
      "2018-07-25T21:19:37.663968: Epoch   3 Batch 1705/3125   train_loss = 0.909\n",
      "2018-07-25T21:19:38.774032: Epoch   3 Batch 1725/3125   train_loss = 0.854\n",
      "2018-07-25T21:19:39.809091: Epoch   3 Batch 1745/3125   train_loss = 0.807\n",
      "2018-07-25T21:19:40.998159: Epoch   3 Batch 1765/3125   train_loss = 0.895\n",
      "2018-07-25T21:19:42.208228: Epoch   3 Batch 1785/3125   train_loss = 0.987\n",
      "2018-07-25T21:19:43.301291: Epoch   3 Batch 1805/3125   train_loss = 0.888\n",
      "2018-07-25T21:19:44.467357: Epoch   3 Batch 1825/3125   train_loss = 0.957\n",
      "2018-07-25T21:19:45.590422: Epoch   3 Batch 1845/3125   train_loss = 0.933\n",
      "2018-07-25T21:19:46.724487: Epoch   3 Batch 1865/3125   train_loss = 0.783\n",
      "2018-07-25T21:19:47.851551: Epoch   3 Batch 1885/3125   train_loss = 0.911\n",
      "2018-07-25T21:19:49.060620: Epoch   3 Batch 1905/3125   train_loss = 0.827\n",
      "2018-07-25T21:19:50.237687: Epoch   3 Batch 1925/3125   train_loss = 0.820\n",
      "2018-07-25T21:19:51.289748: Epoch   3 Batch 1945/3125   train_loss = 0.948\n",
      "2018-07-25T21:19:52.448814: Epoch   3 Batch 1965/3125   train_loss = 0.850\n",
      "2018-07-25T21:19:53.649883: Epoch   3 Batch 1985/3125   train_loss = 0.862\n",
      "2018-07-25T21:19:55.030962: Epoch   3 Batch 2005/3125   train_loss = 0.875\n",
      "2018-07-25T21:19:56.178027: Epoch   3 Batch 2025/3125   train_loss = 0.965\n",
      "2018-07-25T21:19:57.380096: Epoch   3 Batch 2045/3125   train_loss = 0.771\n",
      "2018-07-25T21:19:58.547163: Epoch   3 Batch 2065/3125   train_loss = 0.706\n",
      "2018-07-25T21:19:59.726230: Epoch   3 Batch 2085/3125   train_loss = 1.011\n",
      "2018-07-25T21:20:00.904298: Epoch   3 Batch 2105/3125   train_loss = 0.832\n",
      "2018-07-25T21:20:01.999360: Epoch   3 Batch 2125/3125   train_loss = 1.018\n",
      "2018-07-25T21:20:03.173427: Epoch   3 Batch 2145/3125   train_loss = 0.998\n",
      "2018-07-25T21:20:04.389497: Epoch   3 Batch 2165/3125   train_loss = 0.845\n",
      "2018-07-25T21:20:05.550563: Epoch   3 Batch 2185/3125   train_loss = 0.949\n",
      "2018-07-25T21:20:06.686628: Epoch   3 Batch 2205/3125   train_loss = 0.924\n",
      "2018-07-25T21:20:07.924699: Epoch   3 Batch 2225/3125   train_loss = 0.831\n",
      "2018-07-25T21:20:09.066764: Epoch   3 Batch 2245/3125   train_loss = 0.807\n",
      "2018-07-25T21:20:10.246832: Epoch   3 Batch 2265/3125   train_loss = 0.879\n",
      "2018-07-25T21:20:11.429900: Epoch   3 Batch 2285/3125   train_loss = 0.996\n",
      "2018-07-25T21:20:12.589966: Epoch   3 Batch 2305/3125   train_loss = 0.804\n",
      "2018-07-25T21:20:13.754033: Epoch   3 Batch 2325/3125   train_loss = 0.823\n",
      "2018-07-25T21:20:14.852095: Epoch   3 Batch 2345/3125   train_loss = 0.855\n",
      "2018-07-25T21:20:15.933157: Epoch   3 Batch 2365/3125   train_loss = 0.744\n",
      "2018-07-25T21:20:17.053221: Epoch   3 Batch 2385/3125   train_loss = 0.917\n",
      "2018-07-25T21:20:18.127283: Epoch   3 Batch 2405/3125   train_loss = 0.852\n",
      "2018-07-25T21:20:19.193344: Epoch   3 Batch 2425/3125   train_loss = 0.856\n",
      "2018-07-25T21:20:20.306407: Epoch   3 Batch 2445/3125   train_loss = 0.916\n",
      "2018-07-25T21:20:21.502476: Epoch   3 Batch 2465/3125   train_loss = 0.782\n",
      "2018-07-25T21:20:22.668542: Epoch   3 Batch 2485/3125   train_loss = 0.848\n",
      "2018-07-25T21:20:23.815608: Epoch   3 Batch 2505/3125   train_loss = 0.880\n",
      "2018-07-25T21:20:24.962674: Epoch   3 Batch 2525/3125   train_loss = 0.874\n",
      "2018-07-25T21:20:26.178743: Epoch   3 Batch 2545/3125   train_loss = 1.060\n",
      "2018-07-25T21:20:27.352810: Epoch   3 Batch 2565/3125   train_loss = 0.861\n",
      "2018-07-25T21:20:28.490875: Epoch   3 Batch 2585/3125   train_loss = 0.749\n",
      "2018-07-25T21:20:29.637941: Epoch   3 Batch 2605/3125   train_loss = 0.845\n",
      "2018-07-25T21:20:30.844010: Epoch   3 Batch 2625/3125   train_loss = 0.990\n",
      "2018-07-25T21:20:32.015077: Epoch   3 Batch 2645/3125   train_loss = 0.930\n",
      "2018-07-25T21:20:33.180144: Epoch   3 Batch 2665/3125   train_loss = 1.007\n",
      "2018-07-25T21:20:34.274206: Epoch   3 Batch 2685/3125   train_loss = 0.880\n",
      "2018-07-25T21:20:35.430272: Epoch   3 Batch 2705/3125   train_loss = 0.784\n",
      "2018-07-25T21:20:36.618340: Epoch   3 Batch 2725/3125   train_loss = 0.965\n",
      "2018-07-25T21:20:37.755405: Epoch   3 Batch 2745/3125   train_loss = 0.911\n",
      "2018-07-25T21:20:38.898471: Epoch   3 Batch 2765/3125   train_loss = 0.796\n",
      "2018-07-25T21:20:40.024535: Epoch   3 Batch 2785/3125   train_loss = 1.017\n",
      "2018-07-25T21:20:41.146599: Epoch   3 Batch 2805/3125   train_loss = 0.872\n",
      "2018-07-25T21:20:42.274664: Epoch   3 Batch 2825/3125   train_loss = 0.845\n",
      "2018-07-25T21:20:43.387727: Epoch   3 Batch 2845/3125   train_loss = 0.873\n",
      "2018-07-25T21:20:44.491791: Epoch   3 Batch 2865/3125   train_loss = 0.842\n",
      "2018-07-25T21:20:45.626856: Epoch   3 Batch 2885/3125   train_loss = 0.894\n",
      "2018-07-25T21:20:46.737919: Epoch   3 Batch 2905/3125   train_loss = 0.932\n",
      "2018-07-25T21:20:47.901986: Epoch   3 Batch 2925/3125   train_loss = 0.898\n",
      "2018-07-25T21:20:49.032050: Epoch   3 Batch 2945/3125   train_loss = 0.941\n",
      "2018-07-25T21:20:50.080110: Epoch   3 Batch 2965/3125   train_loss = 0.985\n",
      "2018-07-25T21:20:51.206175: Epoch   3 Batch 2985/3125   train_loss = 0.839\n",
      "2018-07-25T21:20:52.402243: Epoch   3 Batch 3005/3125   train_loss = 0.834\n",
      "2018-07-25T21:20:53.586311: Epoch   3 Batch 3025/3125   train_loss = 0.964\n",
      "2018-07-25T21:20:54.645371: Epoch   3 Batch 3045/3125   train_loss = 0.894\n",
      "2018-07-25T21:20:55.778436: Epoch   3 Batch 3065/3125   train_loss = 0.838\n",
      "2018-07-25T21:20:58.570596: Epoch   3 Batch 3085/3125   train_loss = 0.918\n",
      "2018-07-25T21:20:59.676659: Epoch   3 Batch 3105/3125   train_loss = 0.893\n",
      "2018-07-25T21:21:01.114741: Epoch   3 Batch   17/781   test_loss = 0.884\n",
      "2018-07-25T21:21:01.448760: Epoch   3 Batch   37/781   test_loss = 0.871\n",
      "2018-07-25T21:21:01.766779: Epoch   3 Batch   57/781   test_loss = 0.991\n",
      "2018-07-25T21:21:02.082797: Epoch   3 Batch   77/781   test_loss = 0.915\n",
      "2018-07-25T21:21:02.400815: Epoch   3 Batch   97/781   test_loss = 0.759\n",
      "2018-07-25T21:21:02.711833: Epoch   3 Batch  117/781   test_loss = 0.897\n",
      "2018-07-25T21:21:03.018850: Epoch   3 Batch  137/781   test_loss = 0.936\n",
      "2018-07-25T21:21:03.332868: Epoch   3 Batch  157/781   test_loss = 0.894\n",
      "2018-07-25T21:21:03.643886: Epoch   3 Batch  177/781   test_loss = 0.878\n",
      "2018-07-25T21:21:03.966905: Epoch   3 Batch  197/781   test_loss = 0.901\n",
      "2018-07-25T21:21:04.284923: Epoch   3 Batch  217/781   test_loss = 0.735\n",
      "2018-07-25T21:21:04.602941: Epoch   3 Batch  237/781   test_loss = 0.745\n",
      "2018-07-25T21:21:04.912959: Epoch   3 Batch  257/781   test_loss = 1.066\n",
      "2018-07-25T21:21:05.255978: Epoch   3 Batch  277/781   test_loss = 0.941\n",
      "2018-07-25T21:21:05.569996: Epoch   3 Batch  297/781   test_loss = 1.014\n",
      "2018-07-25T21:21:05.885014: Epoch   3 Batch  317/781   test_loss = 1.060\n",
      "2018-07-25T21:21:06.199032: Epoch   3 Batch  337/781   test_loss = 0.923\n",
      "2018-07-25T21:21:06.508050: Epoch   3 Batch  357/781   test_loss = 0.917\n",
      "2018-07-25T21:21:06.818068: Epoch   3 Batch  377/781   test_loss = 0.963\n",
      "2018-07-25T21:21:07.121085: Epoch   3 Batch  397/781   test_loss = 0.949\n",
      "2018-07-25T21:21:07.422102: Epoch   3 Batch  417/781   test_loss = 0.808\n",
      "2018-07-25T21:21:07.722119: Epoch   3 Batch  437/781   test_loss = 0.808\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-07-25T21:21:08.013136: Epoch   3 Batch  457/781   test_loss = 0.739\n",
      "2018-07-25T21:21:08.323154: Epoch   3 Batch  477/781   test_loss = 0.907\n",
      "2018-07-25T21:21:08.620171: Epoch   3 Batch  497/781   test_loss = 0.801\n",
      "2018-07-25T21:21:08.913187: Epoch   3 Batch  517/781   test_loss = 0.810\n",
      "2018-07-25T21:21:09.196204: Epoch   3 Batch  537/781   test_loss = 0.846\n",
      "2018-07-25T21:21:09.474220: Epoch   3 Batch  557/781   test_loss = 1.028\n",
      "2018-07-25T21:21:09.778237: Epoch   3 Batch  577/781   test_loss = 0.992\n",
      "2018-07-25T21:21:10.073254: Epoch   3 Batch  597/781   test_loss = 0.849\n",
      "2018-07-25T21:21:10.370271: Epoch   3 Batch  617/781   test_loss = 0.931\n",
      "2018-07-25T21:21:10.659287: Epoch   3 Batch  637/781   test_loss = 0.840\n",
      "2018-07-25T21:21:10.953304: Epoch   3 Batch  657/781   test_loss = 0.998\n",
      "2018-07-25T21:21:11.252321: Epoch   3 Batch  677/781   test_loss = 0.907\n",
      "2018-07-25T21:21:11.532337: Epoch   3 Batch  697/781   test_loss = 0.933\n",
      "2018-07-25T21:21:11.825354: Epoch   3 Batch  717/781   test_loss = 0.848\n",
      "2018-07-25T21:21:12.119371: Epoch   3 Batch  737/781   test_loss = 0.752\n",
      "2018-07-25T21:21:12.414388: Epoch   3 Batch  757/781   test_loss = 1.068\n",
      "2018-07-25T21:21:12.715405: Epoch   3 Batch  777/781   test_loss = 0.963\n",
      "2018-07-25T21:21:13.443447: Epoch   4 Batch    0/3125   train_loss = 0.965\n",
      "2018-07-25T21:21:14.572511: Epoch   4 Batch   20/3125   train_loss = 0.844\n",
      "2018-07-25T21:21:15.684575: Epoch   4 Batch   40/3125   train_loss = 0.873\n",
      "2018-07-25T21:21:16.810639: Epoch   4 Batch   60/3125   train_loss = 0.753\n",
      "2018-07-25T21:21:17.956705: Epoch   4 Batch   80/3125   train_loss = 0.901\n",
      "2018-07-25T21:21:19.069768: Epoch   4 Batch  100/3125   train_loss = 0.944\n",
      "2018-07-25T21:21:20.207833: Epoch   4 Batch  120/3125   train_loss = 0.926\n",
      "2018-07-25T21:21:21.344898: Epoch   4 Batch  140/3125   train_loss = 0.901\n",
      "2018-07-25T21:21:22.448962: Epoch   4 Batch  160/3125   train_loss = 0.752\n",
      "2018-07-25T21:21:23.501022: Epoch   4 Batch  180/3125   train_loss = 0.888\n",
      "2018-07-25T21:21:24.532081: Epoch   4 Batch  200/3125   train_loss = 1.123\n",
      "2018-07-25T21:21:25.644144: Epoch   4 Batch  220/3125   train_loss = 0.891\n",
      "2018-07-25T21:21:26.801211: Epoch   4 Batch  240/3125   train_loss = 0.938\n",
      "2018-07-25T21:21:27.898273: Epoch   4 Batch  260/3125   train_loss = 0.966\n",
      "2018-07-25T21:21:28.997336: Epoch   4 Batch  280/3125   train_loss = 1.101\n",
      "2018-07-25T21:21:30.051396: Epoch   4 Batch  300/3125   train_loss = 1.011\n",
      "2018-07-25T21:21:31.127458: Epoch   4 Batch  320/3125   train_loss = 0.987\n",
      "2018-07-25T21:21:32.185519: Epoch   4 Batch  340/3125   train_loss = 0.753\n",
      "2018-07-25T21:21:34.886673: Epoch   4 Batch  360/3125   train_loss = 0.835\n",
      "2018-07-25T21:21:35.938733: Epoch   4 Batch  380/3125   train_loss = 0.851\n",
      "2018-07-25T21:21:37.057797: Epoch   4 Batch  400/3125   train_loss = 0.869\n",
      "2018-07-25T21:21:38.172861: Epoch   4 Batch  420/3125   train_loss = 0.819\n",
      "2018-07-25T21:21:39.229921: Epoch   4 Batch  440/3125   train_loss = 0.897\n",
      "2018-07-25T21:21:43.877187: Epoch   4 Batch  460/3125   train_loss = 0.913\n",
      "2018-07-25T21:21:45.032253: Epoch   4 Batch  480/3125   train_loss = 0.987\n",
      "2018-07-25T21:21:47.909418: Epoch   4 Batch  500/3125   train_loss = 0.681\n",
      "2018-07-25T21:21:52.154661: Epoch   4 Batch  520/3125   train_loss = 0.920\n",
      "2018-07-25T21:21:54.709807: Epoch   4 Batch  540/3125   train_loss = 0.785\n",
      "2018-07-25T21:21:57.283954: Epoch   4 Batch  560/3125   train_loss = 0.988\n",
      "2018-07-25T21:21:58.419019: Epoch   4 Batch  580/3125   train_loss = 0.963\n",
      "2018-07-25T21:21:59.609087: Epoch   4 Batch  600/3125   train_loss = 0.892\n",
      "2018-07-25T21:22:02.919276: Epoch   4 Batch  620/3125   train_loss = 0.913\n",
      "2018-07-25T21:22:04.034340: Epoch   4 Batch  640/3125   train_loss = 0.859\n",
      "2018-07-25T21:22:07.024511: Epoch   4 Batch  660/3125   train_loss = 0.866\n",
      "2018-07-25T21:22:08.254582: Epoch   4 Batch  680/3125   train_loss = 0.892\n",
      "2018-07-25T21:22:09.593658: Epoch   4 Batch  700/3125   train_loss = 0.933\n",
      "2018-07-25T21:22:10.772726: Epoch   4 Batch  720/3125   train_loss = 0.825\n",
      "2018-07-25T21:22:12.202807: Epoch   4 Batch  740/3125   train_loss = 0.935\n",
      "2018-07-25T21:22:13.287869: Epoch   4 Batch  760/3125   train_loss = 0.788\n",
      "2018-07-25T21:22:14.450936: Epoch   4 Batch  780/3125   train_loss = 0.922\n",
      "2018-07-25T21:22:15.509997: Epoch   4 Batch  800/3125   train_loss = 0.809\n",
      "2018-07-25T21:22:16.573057: Epoch   4 Batch  820/3125   train_loss = 0.861\n",
      "2018-07-25T21:22:17.754125: Epoch   4 Batch  840/3125   train_loss = 0.855\n",
      "2018-07-25T21:22:18.818186: Epoch   4 Batch  860/3125   train_loss = 0.814\n",
      "2018-07-25T21:22:19.915249: Epoch   4 Batch  880/3125   train_loss = 0.775\n",
      "2018-07-25T21:22:21.022312: Epoch   4 Batch  900/3125   train_loss = 0.891\n",
      "2018-07-25T21:22:22.068372: Epoch   4 Batch  920/3125   train_loss = 0.946\n",
      "2018-07-25T21:22:23.210437: Epoch   4 Batch  940/3125   train_loss = 0.857\n",
      "2018-07-25T21:22:24.440507: Epoch   4 Batch  960/3125   train_loss = 0.924\n",
      "2018-07-25T21:22:25.568572: Epoch   4 Batch  980/3125   train_loss = 0.987\n",
      "2018-07-25T21:22:27.745696: Epoch   4 Batch 1000/3125   train_loss = 0.944\n",
      "2018-07-25T21:22:29.063772: Epoch   4 Batch 1020/3125   train_loss = 0.890\n",
      "2018-07-25T21:22:30.237839: Epoch   4 Batch 1040/3125   train_loss = 0.763\n",
      "2018-07-25T21:22:31.420907: Epoch   4 Batch 1060/3125   train_loss = 0.976\n",
      "2018-07-25T21:22:32.672978: Epoch   4 Batch 1080/3125   train_loss = 0.914\n",
      "2018-07-25T21:22:33.932050: Epoch   4 Batch 1100/3125   train_loss = 0.862\n",
      "2018-07-25T21:22:35.344131: Epoch   4 Batch 1120/3125   train_loss = 0.882\n",
      "2018-07-25T21:22:36.510198: Epoch   4 Batch 1140/3125   train_loss = 0.838\n",
      "2018-07-25T21:22:37.678264: Epoch   4 Batch 1160/3125   train_loss = 0.783\n",
      "2018-07-25T21:22:38.826330: Epoch   4 Batch 1180/3125   train_loss = 0.807\n",
      "2018-07-25T21:22:39.987397: Epoch   4 Batch 1200/3125   train_loss = 0.956\n",
      "2018-07-25T21:22:41.168464: Epoch   4 Batch 1220/3125   train_loss = 0.954\n",
      "2018-07-25T21:22:42.328530: Epoch   4 Batch 1240/3125   train_loss = 0.743\n",
      "2018-07-25T21:22:43.468596: Epoch   4 Batch 1260/3125   train_loss = 0.853\n",
      "2018-07-25T21:22:44.669664: Epoch   4 Batch 1280/3125   train_loss = 0.939\n",
      "2018-07-25T21:22:45.895735: Epoch   4 Batch 1300/3125   train_loss = 0.828\n",
      "2018-07-25T21:22:47.106804: Epoch   4 Batch 1320/3125   train_loss = 0.849\n",
      "2018-07-25T21:22:48.270870: Epoch   4 Batch 1340/3125   train_loss = 0.783\n",
      "2018-07-25T21:22:49.465939: Epoch   4 Batch 1360/3125   train_loss = 0.833\n",
      "2018-07-25T21:22:50.598003: Epoch   4 Batch 1380/3125   train_loss = 0.792\n",
      "2018-07-25T21:22:51.771071: Epoch   4 Batch 1400/3125   train_loss = 0.926\n",
      "2018-07-25T21:22:52.949138: Epoch   4 Batch 1420/3125   train_loss = 0.907\n",
      "2018-07-25T21:22:54.079203: Epoch   4 Batch 1440/3125   train_loss = 0.746\n",
      "2018-07-25T21:22:55.261270: Epoch   4 Batch 1460/3125   train_loss = 0.913\n",
      "2018-07-25T21:22:56.390335: Epoch   4 Batch 1480/3125   train_loss = 0.878\n",
      "2018-07-25T21:22:57.595404: Epoch   4 Batch 1500/3125   train_loss = 0.886\n",
      "2018-07-25T21:22:58.685466: Epoch   4 Batch 1520/3125   train_loss = 0.809\n",
      "2018-07-25T21:22:59.862533: Epoch   4 Batch 1540/3125   train_loss = 0.965\n",
      "2018-07-25T21:23:00.990598: Epoch   4 Batch 1560/3125   train_loss = 0.784\n",
      "2018-07-25T21:23:02.180666: Epoch   4 Batch 1580/3125   train_loss = 0.971\n",
      "2018-07-25T21:23:03.356733: Epoch   4 Batch 1600/3125   train_loss = 0.816\n",
      "2018-07-25T21:23:04.563802: Epoch   4 Batch 1620/3125   train_loss = 0.789\n",
      "2018-07-25T21:23:05.758871: Epoch   4 Batch 1640/3125   train_loss = 0.941\n",
      "2018-07-25T21:23:06.911937: Epoch   4 Batch 1660/3125   train_loss = 1.036\n",
      "2018-07-25T21:23:08.121006: Epoch   4 Batch 1680/3125   train_loss = 0.902\n",
      "2018-07-25T21:23:09.346076: Epoch   4 Batch 1700/3125   train_loss = 0.786\n",
      "2018-07-25T21:23:10.562145: Epoch   4 Batch 1720/3125   train_loss = 0.902\n",
      "2018-07-25T21:23:11.724212: Epoch   4 Batch 1740/3125   train_loss = 0.925\n",
      "2018-07-25T21:23:12.903279: Epoch   4 Batch 1760/3125   train_loss = 0.853\n",
      "2018-07-25T21:23:14.026343: Epoch   4 Batch 1780/3125   train_loss = 0.906\n",
      "2018-07-25T21:23:15.107405: Epoch   4 Batch 1800/3125   train_loss = 0.832\n",
      "2018-07-25T21:23:16.199468: Epoch   4 Batch 1820/3125   train_loss = 0.830\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-07-25T21:23:17.364534: Epoch   4 Batch 1840/3125   train_loss = 0.909\n",
      "2018-07-25T21:23:18.482598: Epoch   4 Batch 1860/3125   train_loss = 0.995\n",
      "2018-07-25T21:23:19.708669: Epoch   4 Batch 1880/3125   train_loss = 0.862\n",
      "2018-07-25T21:23:20.903737: Epoch   4 Batch 1900/3125   train_loss = 0.734\n",
      "2018-07-25T21:23:22.074804: Epoch   4 Batch 1920/3125   train_loss = 0.898\n",
      "2018-07-25T21:23:23.246871: Epoch   4 Batch 1940/3125   train_loss = 0.785\n",
      "2018-07-25T21:23:24.449940: Epoch   4 Batch 1960/3125   train_loss = 0.798\n",
      "2018-07-25T21:23:25.647008: Epoch   4 Batch 1980/3125   train_loss = 0.883\n",
      "2018-07-25T21:23:26.750071: Epoch   4 Batch 2000/3125   train_loss = 1.056\n",
      "2018-07-25T21:23:27.937139: Epoch   4 Batch 2020/3125   train_loss = 0.909\n",
      "2018-07-25T21:23:29.128207: Epoch   4 Batch 2040/3125   train_loss = 0.788\n",
      "2018-07-25T21:23:30.242271: Epoch   4 Batch 2060/3125   train_loss = 0.794\n",
      "2018-07-25T21:23:31.349334: Epoch   4 Batch 2080/3125   train_loss = 0.977\n",
      "2018-07-25T21:23:32.516401: Epoch   4 Batch 2100/3125   train_loss = 0.801\n",
      "2018-07-25T21:23:33.671467: Epoch   4 Batch 2120/3125   train_loss = 0.819\n",
      "2018-07-25T21:23:34.798532: Epoch   4 Batch 2140/3125   train_loss = 0.834\n",
      "2018-07-25T21:23:35.960598: Epoch   4 Batch 2160/3125   train_loss = 0.823\n",
      "2018-07-25T21:23:37.079662: Epoch   4 Batch 2180/3125   train_loss = 0.922\n",
      "2018-07-25T21:23:38.261730: Epoch   4 Batch 2200/3125   train_loss = 0.795\n",
      "2018-07-25T21:23:39.385794: Epoch   4 Batch 2220/3125   train_loss = 0.842\n",
      "2018-07-25T21:23:40.572862: Epoch   4 Batch 2240/3125   train_loss = 0.862\n",
      "2018-07-25T21:23:41.845935: Epoch   4 Batch 2260/3125   train_loss = 0.867\n",
      "2018-07-25T21:23:43.024002: Epoch   4 Batch 2280/3125   train_loss = 0.876\n",
      "2018-07-25T21:23:44.114064: Epoch   4 Batch 2300/3125   train_loss = 0.854\n",
      "2018-07-25T21:23:45.184126: Epoch   4 Batch 2320/3125   train_loss = 0.926\n",
      "2018-07-25T21:23:46.293189: Epoch   4 Batch 2340/3125   train_loss = 0.878\n",
      "2018-07-25T21:23:47.395252: Epoch   4 Batch 2360/3125   train_loss = 0.849\n",
      "2018-07-25T21:23:48.532317: Epoch   4 Batch 2380/3125   train_loss = 0.843\n",
      "2018-07-25T21:23:49.644381: Epoch   4 Batch 2400/3125   train_loss = 0.965\n",
      "2018-07-25T21:23:50.769445: Epoch   4 Batch 2420/3125   train_loss = 0.804\n",
      "2018-07-25T21:23:51.897510: Epoch   4 Batch 2440/3125   train_loss = 0.841\n",
      "2018-07-25T21:23:53.042575: Epoch   4 Batch 2460/3125   train_loss = 0.852\n",
      "2018-07-25T21:23:54.769674: Epoch   4 Batch 2480/3125   train_loss = 0.965\n",
      "2018-07-25T21:23:56.116751: Epoch   4 Batch 2500/3125   train_loss = 0.838\n",
      "2018-07-25T21:23:57.317820: Epoch   4 Batch 2520/3125   train_loss = 0.941\n",
      "2018-07-25T21:23:58.561891: Epoch   4 Batch 2540/3125   train_loss = 0.802\n",
      "2018-07-25T21:23:59.753959: Epoch   4 Batch 2560/3125   train_loss = 0.659\n",
      "2018-07-25T21:24:00.905025: Epoch   4 Batch 2580/3125   train_loss = 0.856\n",
      "2018-07-25T21:24:02.111094: Epoch   4 Batch 2600/3125   train_loss = 0.866\n",
      "2018-07-25T21:24:03.237158: Epoch   4 Batch 2620/3125   train_loss = 0.792\n",
      "2018-07-25T21:24:04.413225: Epoch   4 Batch 2640/3125   train_loss = 0.844\n",
      "2018-07-25T21:24:05.580292: Epoch   4 Batch 2660/3125   train_loss = 1.056\n",
      "2018-07-25T21:24:06.747359: Epoch   4 Batch 2680/3125   train_loss = 0.770\n",
      "2018-07-25T21:24:07.874423: Epoch   4 Batch 2700/3125   train_loss = 0.836\n",
      "2018-07-25T21:24:09.024489: Epoch   4 Batch 2720/3125   train_loss = 0.753\n",
      "2018-07-25T21:24:10.113551: Epoch   4 Batch 2740/3125   train_loss = 0.821\n",
      "2018-07-25T21:24:11.250617: Epoch   4 Batch 2760/3125   train_loss = 0.774\n",
      "2018-07-25T21:24:12.369681: Epoch   4 Batch 2780/3125   train_loss = 0.875\n",
      "2018-07-25T21:24:13.516746: Epoch   4 Batch 2800/3125   train_loss = 1.008\n",
      "2018-07-25T21:24:14.706814: Epoch   4 Batch 2820/3125   train_loss = 1.015\n",
      "2018-07-25T21:24:15.835879: Epoch   4 Batch 2840/3125   train_loss = 0.832\n",
      "2018-07-25T21:24:16.977944: Epoch   4 Batch 2860/3125   train_loss = 0.793\n",
      "2018-07-25T21:24:18.090008: Epoch   4 Batch 2880/3125   train_loss = 0.878\n",
      "2018-07-25T21:24:19.251074: Epoch   4 Batch 2900/3125   train_loss = 0.890\n",
      "2018-07-25T21:24:20.333136: Epoch   4 Batch 2920/3125   train_loss = 0.815\n",
      "2018-07-25T21:24:21.554206: Epoch   4 Batch 2940/3125   train_loss = 0.930\n",
      "2018-07-25T21:24:22.655269: Epoch   4 Batch 2960/3125   train_loss = 0.869\n",
      "2018-07-25T21:24:23.759332: Epoch   4 Batch 2980/3125   train_loss = 0.855\n",
      "2018-07-25T21:24:28.120581: Epoch   4 Batch 3000/3125   train_loss = 0.992\n",
      "2018-07-25T21:24:29.250646: Epoch   4 Batch 3020/3125   train_loss = 0.992\n",
      "2018-07-25T21:24:30.377711: Epoch   4 Batch 3040/3125   train_loss = 0.962\n",
      "2018-07-25T21:24:32.795849: Epoch   4 Batch 3060/3125   train_loss = 0.814\n",
      "2018-07-25T21:24:34.427942: Epoch   4 Batch 3080/3125   train_loss = 0.983\n",
      "2018-07-25T21:24:36.204044: Epoch   4 Batch 3100/3125   train_loss = 1.028\n",
      "2018-07-25T21:24:39.961259: Epoch   4 Batch 3120/3125   train_loss = 0.832\n",
      "2018-07-25T21:24:41.225331: Epoch   4 Batch   16/781   test_loss = 0.838\n",
      "2018-07-25T21:24:41.539349: Epoch   4 Batch   36/781   test_loss = 0.945\n",
      "2018-07-25T21:24:41.866368: Epoch   4 Batch   56/781   test_loss = 0.976\n",
      "2018-07-25T21:24:42.177385: Epoch   4 Batch   76/781   test_loss = 0.975\n",
      "2018-07-25T21:24:42.481403: Epoch   4 Batch   96/781   test_loss = 1.038\n",
      "2018-07-25T21:24:42.793421: Epoch   4 Batch  116/781   test_loss = 0.863\n",
      "2018-07-25T21:24:43.117439: Epoch   4 Batch  136/781   test_loss = 0.843\n",
      "2018-07-25T21:24:43.416456: Epoch   4 Batch  156/781   test_loss = 0.894\n",
      "2018-07-25T21:24:43.698472: Epoch   4 Batch  176/781   test_loss = 0.856\n",
      "2018-07-25T21:24:43.995489: Epoch   4 Batch  196/781   test_loss = 0.806\n",
      "2018-07-25T21:24:44.288506: Epoch   4 Batch  216/781   test_loss = 0.977\n",
      "2018-07-25T21:24:44.609525: Epoch   4 Batch  236/781   test_loss = 0.780\n",
      "2018-07-25T21:24:44.921542: Epoch   4 Batch  256/781   test_loss = 0.848\n",
      "2018-07-25T21:24:45.226560: Epoch   4 Batch  276/781   test_loss = 1.110\n",
      "2018-07-25T21:24:45.541578: Epoch   4 Batch  296/781   test_loss = 0.835\n",
      "2018-07-25T21:24:45.845595: Epoch   4 Batch  316/781   test_loss = 0.876\n",
      "2018-07-25T21:24:46.142612: Epoch   4 Batch  336/781   test_loss = 0.773\n",
      "2018-07-25T21:24:46.435629: Epoch   4 Batch  356/781   test_loss = 0.871\n",
      "2018-07-25T21:24:46.729646: Epoch   4 Batch  376/781   test_loss = 0.914\n",
      "2018-07-25T21:24:47.105667: Epoch   4 Batch  396/781   test_loss = 0.860\n",
      "2018-07-25T21:24:47.406685: Epoch   4 Batch  416/781   test_loss = 0.940\n",
      "2018-07-25T21:24:47.707702: Epoch   4 Batch  436/781   test_loss = 0.905\n",
      "2018-07-25T21:24:48.002719: Epoch   4 Batch  456/781   test_loss = 0.754\n",
      "2018-07-25T21:24:48.335738: Epoch   4 Batch  476/781   test_loss = 0.911\n",
      "2018-07-25T21:24:48.643755: Epoch   4 Batch  496/781   test_loss = 0.951\n",
      "2018-07-25T21:24:48.944773: Epoch   4 Batch  516/781   test_loss = 0.755\n",
      "2018-07-25T21:24:49.244790: Epoch   4 Batch  536/781   test_loss = 0.983\n",
      "2018-07-25T21:24:49.572808: Epoch   4 Batch  556/781   test_loss = 0.833\n",
      "2018-07-25T21:24:49.895827: Epoch   4 Batch  576/781   test_loss = 0.999\n",
      "2018-07-25T21:24:50.186844: Epoch   4 Batch  596/781   test_loss = 1.006\n",
      "2018-07-25T21:24:50.472860: Epoch   4 Batch  616/781   test_loss = 0.932\n",
      "2018-07-25T21:24:50.762877: Epoch   4 Batch  636/781   test_loss = 0.854\n",
      "2018-07-25T21:24:51.052893: Epoch   4 Batch  656/781   test_loss = 0.932\n",
      "2018-07-25T21:24:51.338909: Epoch   4 Batch  676/781   test_loss = 1.103\n",
      "2018-07-25T21:24:51.624926: Epoch   4 Batch  696/781   test_loss = 0.884\n",
      "2018-07-25T21:24:51.939944: Epoch   4 Batch  716/781   test_loss = 0.893\n",
      "2018-07-25T21:24:52.246961: Epoch   4 Batch  736/781   test_loss = 1.035\n",
      "2018-07-25T21:24:52.553979: Epoch   4 Batch  756/781   test_loss = 0.871\n",
      "2018-07-25T21:24:52.855996: Epoch   4 Batch  776/781   test_loss = 0.732\n",
      "Model Trained and Saved\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "losses = {'train':[], 'test':[]}\n",
    "\n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    \n",
    "    #搜集数据给tensorBoard用\n",
    "    # Keep track of gradient values and sparsity\n",
    "    grad_summaries = []\n",
    "    for g, v in gradients:\n",
    "        if g is not None:\n",
    "            grad_hist_summary = tf.summary.histogram(\"{}/grad/hist\".format(v.name.replace(':', '_')), g)\n",
    "            sparsity_summary = tf.summary.scalar(\"{}/grad/sparsity\".format(v.name.replace(':', '_')), tf.nn.zero_fraction(g))\n",
    "            grad_summaries.append(grad_hist_summary)\n",
    "            grad_summaries.append(sparsity_summary)\n",
    "    grad_summaries_merged = tf.summary.merge(grad_summaries)\n",
    "        \n",
    "    # Output directory for models and summaries\n",
    "    timestamp = str(int(time.time()))\n",
    "    out_dir = os.path.abspath(os.path.join(os.path.curdir, \"runs\", timestamp))\n",
    "    print(\"Writing to {}\\n\".format(out_dir))\n",
    "     \n",
    "    # Summaries for loss and accuracy\n",
    "    loss_summary = tf.summary.scalar(\"loss\", loss)\n",
    "\n",
    "    # Train Summaries\n",
    "    train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged])\n",
    "    train_summary_dir = os.path.join(out_dir, \"summaries\", \"train\")\n",
    "    train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)\n",
    "\n",
    "    # Inference summaries\n",
    "    inference_summary_op = tf.summary.merge([loss_summary])\n",
    "    inference_summary_dir = os.path.join(out_dir, \"summaries\", \"inference\")\n",
    "    inference_summary_writer = tf.summary.FileWriter(inference_summary_dir, sess.graph)\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    saver = tf.train.Saver()\n",
    "    for epoch_i in range(num_epochs):\n",
    "        \n",
    "        #将数据集分成训练集和测试集，随机种子不固定\n",
    "        train_X,test_X, train_y, test_y = train_test_split(features,  \n",
    "                                                           targets_values,  \n",
    "                                                           test_size = 0.2,  \n",
    "                                                           random_state = 0)  \n",
    "        \n",
    "        train_batches = get_batches(train_X, train_y, batch_size)\n",
    "        test_batches = get_batches(test_X, test_y, batch_size)\n",
    "    \n",
    "        #训练的迭代，保存训练损失\n",
    "        for batch_i in range(len(train_X) // batch_size):\n",
    "            x, y = next(train_batches)\n",
    "\n",
    "            categories = np.zeros([batch_size, 18])\n",
    "            for i in range(batch_size):\n",
    "                categories[i] = x.take(6,1)[i]\n",
    "\n",
    "            titles = np.zeros([batch_size, sentences_size])\n",
    "            for i in range(batch_size):\n",
    "                titles[i] = x.take(5,1)[i]\n",
    "\n",
    "            feed = {\n",
    "                uid: np.reshape(x.take(0,1), [batch_size, 1]),\n",
    "                user_gender: np.reshape(x.take(2,1), [batch_size, 1]),\n",
    "                user_age: np.reshape(x.take(3,1), [batch_size, 1]),\n",
    "                user_job: np.reshape(x.take(4,1), [batch_size, 1]),\n",
    "                movie_id: np.reshape(x.take(1,1), [batch_size, 1]),\n",
    "                movie_categories: categories,  #x.take(6,1)\n",
    "                movie_titles: titles,  #x.take(5,1)\n",
    "                targets: np.reshape(y, [batch_size, 1]),\n",
    "                dropout_keep_prob: dropout_keep, #dropout_keep\n",
    "                lr: learning_rate}\n",
    "\n",
    "            step, train_loss, summaries, _ = sess.run([global_step, loss, train_summary_op, train_op], feed)  #cost\n",
    "            losses['train'].append(train_loss)\n",
    "            train_summary_writer.add_summary(summaries, step)  #\n",
    "            \n",
    "            # Show every <show_every_n_batches> batches\n",
    "            if (epoch_i * (len(train_X) // batch_size) + batch_i) % show_every_n_batches == 0:\n",
    "                time_str = datetime.datetime.now().isoformat()\n",
    "                print('{}: Epoch {:>3} Batch {:>4}/{}   train_loss = {:.3f}'.format(\n",
    "                    time_str,\n",
    "                    epoch_i,\n",
    "                    batch_i,\n",
    "                    (len(train_X) // batch_size),\n",
    "                    train_loss))\n",
    "                \n",
    "        #使用测试数据的迭代\n",
    "        for batch_i  in range(len(test_X) // batch_size):\n",
    "            x, y = next(test_batches)\n",
    "            \n",
    "            categories = np.zeros([batch_size, 18])\n",
    "            for i in range(batch_size):\n",
    "                categories[i] = x.take(6,1)[i]\n",
    "\n",
    "            titles = np.zeros([batch_size, sentences_size])\n",
    "            for i in range(batch_size):\n",
    "                titles[i] = x.take(5,1)[i]\n",
    "\n",
    "            feed = {\n",
    "                uid: np.reshape(x.take(0,1), [batch_size, 1]),\n",
    "                user_gender: np.reshape(x.take(2,1), [batch_size, 1]),\n",
    "                user_age: np.reshape(x.take(3,1), [batch_size, 1]),\n",
    "                user_job: np.reshape(x.take(4,1), [batch_size, 1]),\n",
    "                movie_id: np.reshape(x.take(1,1), [batch_size, 1]),\n",
    "                movie_categories: categories,  #x.take(6,1)\n",
    "                movie_titles: titles,  #x.take(5,1)\n",
    "                targets: np.reshape(y, [batch_size, 1]),\n",
    "                dropout_keep_prob: 1,\n",
    "                lr: learning_rate}\n",
    "            \n",
    "            step, test_loss, summaries = sess.run([global_step, loss, inference_summary_op], feed)  #cost\n",
    "\n",
    "            #保存测试损失\n",
    "            losses['test'].append(test_loss)\n",
    "            inference_summary_writer.add_summary(summaries, step)  #\n",
    "\n",
    "            time_str = datetime.datetime.now().isoformat()\n",
    "            if (epoch_i * (len(test_X) // batch_size) + batch_i) % show_every_n_batches == 0:\n",
    "                print('{}: Epoch {:>3} Batch {:>4}/{}   test_loss = {:.3f}'.format(\n",
    "                    time_str,\n",
    "                    epoch_i,\n",
    "                    batch_i,\n",
    "                    (len(test_X) // batch_size),\n",
    "                    test_loss))\n",
    "\n",
    "    # Save Model\n",
    "    saver.save(sess, save_dir)  #, global_step=epoch_i\n",
    "    print('Model Trained and Saved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_params((save_dir))\n",
    "\n",
    "load_dir = load_params()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvIAAAH0CAYAAABfKsnMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd8VfX9x/H3NxMyCISwIWwERGQJAjIEBzipSusoWmfVWgXFVUBp1UIdrfPXqlVRcYOCBQUcyJK9QfYOMxAI2et+f38kuSYkgQD33pOTvJ6PB4+b3HPuuZ9QYt/nez/nc4y1VgAAAADcJcjpAgAAAACcPoI8AAAA4EIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIggDwAAALgQQR4AAABwIYI8AAAA4EIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIggDwAAALhQiNMF+IMxZoekGpJ2OlwKAAAAKrdmko5ba5sH+o0rZZCXVKN69eqx7dq1i3W6EAAAAFReGzZsUEZGhiPvXVmD/M527drFLl++3Ok6AAAAUIl17dpVK1as2OnEe9MjDwAAALgQQR4AAABwIYI8AAAA4EIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuVFnnyAMAgHLweDxKSkpSSkqKsrKyZK11uiTAMcYYhYeHKzo6WrGxsQoKqthr3gR5AACqKI/Hoz179ig9Pd3pUoAKwVqrzMxMZWZmKi0tTU2aNKnQYZ4gDwBAFZWUlKT09HSFhISofv36ioyMrNChBfA3j8ejtLQ0HThwQOnp6UpKSlJcXJzTZZWJ31YAAKqolJQUSVL9+vUVHR1NiEeVFxQUpOjoaNWvX1/Sr78jFRW/sQAAVFFZWVmSpMjISIcrASqWwt+Jwt+RioogDwBAFVV4YSsr8UBxxhhJqvAXf/ObCwAAABRRGOQrOoI8AAAA4EJMrfERa63yPL9+/BISzDkSAAAA/Ie06SOZOR61GvWtWo36Vh3GznS6HAAA4CKpqakyxuiqq64662N169ZNUVFRPqjKd15//XUZYzRp0iSnS6lUCPIAAKDKMsac1p8JEyY4XTLgRWsNAACosp5++ukSz7388stKTk7WQw89pJo1axbb1qlTJ7/UERkZqQ0bNvhkJX3y5MkVfmwifIMg7wcVfFIRAAAoMHbs2BLPTZgwQcnJyRo+fLiaNWsWkDqMMWrbtq1PjtW0aVOfHAcVH601PuKSKUUAAMAHCvvQMzIyNHr0aLVq1UphYWF64IEHJElHjhzR+PHj1a9fPzVs2FBhYWGqV6+err/+eq1YsaLE8crqkR85cqSMMVq2bJk++ugjde3aVdWrV1dcXJyGDRumQ4cOlVlbUdOmTZMxRi+++KKWLFmiyy+/XDExMYqKitIll1yi5cuXl/pz7t69W7///e8VFxeniIgIde3aVZ999lmx452thQsX6tprr1VcXJzCw8PVokULDR8+XImJiSX23bdvnx566CG1adNGERERqlWrltq1a6c777xTe/bs8e7n8Xj09ttvq0ePHoqLi1P16tUVHx+vK664QlOmTDnrmisKVuQBAADOgMfj0VVXXaVNmzbp8ssvV+3atb2r4StXrtTTTz+t/v3769prr1VMTIx27Nihr7/+WtOmTdN3332nvn37lvu9nn/+eU2bNk3XXnutLr74Yi1YsEATJ07UunXrtGzZMgUHB5frOPPnz9fo0aPVv39/3X333dq+fbumTJmi/v37a926dcVW8xMSEtSzZ0/t27dPAwcO1AUXXKC9e/fqtttu0+DBg0/vL6sMn3/+uW655RYFBwdr6NChaty4sRYtWqRXXnlFU6dO1YIFC9SwYUNJ0vHjx9WjRw/t27dPl112mYYMGaKcnBzt2rVLkyZN0rBhw9SkSRNJ0vDhw/Xaa6+pdevWuummmxQVFaV9+/Zp8eLFmjJlioYMGeKT+p1GkAcAADgDGRkZSklJ0bp160r00nfp0kUHDhxQrVq1ij2/bds29ejRQ4888oiWLl1a7vf64YcftGrVKrVp00ZS/tjrIUOG6Ouvv9bMmTN1xRVXlOs4U6dO1RdffKEbbrjB+9xLL72kkSNH6o033tDzzz/vff6RRx7Rvn379Le//U1jxozxPn///ffroosuKnftZUlKStJdd90lY4zmz5+vbt26ebeNGTNGzz77rB544AF9+eWXkqTp06crISFBo0eP1jPPPFPsWJmZmcrNzZX062p8y5YttXbtWoWHhxfb9/Dhw2dde0VBkPcDWuQBAJVBsyemO11Cue0cf6Uj7ztu3LgSIV6SYmNjS92/ZcuWuuaaa/Tee+/pyJEjql27drne59FHH/WGeCm/p/6uu+7S119/rSVLlpQ7yF9++eXFQrwk3XPPPRo5cqSWLFnifS4lJUVffvml6tatq0cffbTY/hdeeKGGDh2qTz/9tFzvWZYvvvhCKSkpuvvuu4uFeEkaNWqU/vvf/2rq1Kk6fPiw4uLivNuqV69e4ljVqlUr9r0xRmFhYaV+UlH0WG5HjzwAAMAZ6t69e5nbZs+ereuuu06NGzdWWFiYd4Tle++9Jym/37u8Tgy6krxtJEePHj2r40RHRysmJqbYcdatW6fc3Fx17dq1REiW5JMV+cJrBQYMGFBiW7Vq1dSrVy95PB6tXr1aknTppZeqTp06GjNmjK666iq98cYbWrVqlTweT7HXBgUF6cYbb9SGDRvUoUMHjRkzRrNmzVJKSspZ11zRsCIPAABwBiIiIhQdHV3qtokTJ+rWW29VVFSULr30UjVv3lyRkZEyxmjWrFlauHDhaY2ILG3VPyQkP8bl5eWd1XEKj1X0OMnJyZKkevXqlbp/Wc+fjsL3aNCgQanbC58/duyYpPyV9MWLF2vs2LGaNm2apk+f7q3lwQcf1OOPP+5dgX/zzTfVtm1bvf/++3r22WclSaGhobrmmmv00ksvVZrJPgR5AABQKqfaVdzCnGRk3ejRoxUdHa2VK1eqRYsWxbZt2bJFCxcu9Hd5Z6VGjRqSpIMHD5a6vaznT0dMTIwk6cCBA6Vu379/f7H9JKl58+Z6//335fF4tG7dOv3www96/fXXNWrUKAUHB+vxxx+XlB/aH3vsMT322GM6cOCA5s2bp4kTJ2ry5MnauHGjVq9eXe4LhCsyn7TWGGNuMMa8ZoyZZ4w5boyxxpiJZezb2hjzuDHmR2PMHmNMtjHmoDFmqjHmYl/U4zia5AEAqLJyc3O1a9cuderUqUSIz8nJqfAhXpLOO+88hYSEaPny5crMzCyxff78+Wf9Hp07d5Yk/fTTTyW2ZWVlaeHChTLGlHoTrqCgIHXs2FEjRozQtGnTJKnMsZL169fX0KFDNXXqVHXv3l3r16/X1q1bz7r+isBXPfKjJT0gqZOkvafY9xlJ4yXVk/SNpJckLZB0paQfjTEP+qimgGKOPAAAkPLbVBo1aqT169cXm5Di8Xj05JNPaseOHQ5WVz7R0dEaMmSIDh06pBdeeKHYtsWLF+uLL7446/f47W9/q6ioKL333nvePvhC48aN0/79+73z5SVp1apVSkhIKHGcwk8HIiIiJOXP5J8zZ06J/bKysrztPKVdMOtGvmqtGSEpQdJWSf0kzT7JvjMk/cNau7Lok8aYfpK+k/SCMeYLa+1+H9UGAAAQUCNGjNDIkSPVsWNHXXfddQoKCtKcOXO0c+dODR48WN9++63TJZ7SSy+9pPnz5+upp57S3LlzdcEFFyghIUGff/65rr76ak2ZMkVBQWe+JhwbG6u33npLw4YNU8+ePTV06FA1atRIixYt0uzZs9WkSRO9/vrr3v2nTZump59+WhdddJHOOeccxcXFadeuXZo6daqCg4M1cuRISfk99f3791fLli3VvXt3xcfHKz09XTNmzNCWLVt08803Kz4+/qz/fioCnwR5a603uJ+sX6xg3wllPD/HGPOTpEsl9ZI02Re1OcHSWwMAQJX28MMPKyoqSq+//rreffddRUZGqn///vr888/19ttvuyLIx8fHa9GiRXryySc1c+ZMzZ8/X+3bt9f777+vjIwMTZkyxdtLf6ZuuukmxcfHa/z48Zo2bZpSUlLUsGFD/fnPf9bo0aNVt25d777XXHONEhMTNW/ePH355ZdKTU1VgwYNdPXVV+uRRx7xTuSpXbu2/v73v2v27NmaN2+eEhMTVaNGDbVu3VqPP/64brvttrOquSIx1vo2dBpj+it/Rf4ja+3vT/O10yVdIWmItXbqWdSwvEuXLl3Kut2wP2TnetRmdP4vZWiw0ZbnyjfPFQAAp2zYsEGS1K5dO4crgds89NBDevXVVzV//nz17t3b6XL8ory/H127dtWKFStWWGu7BqKuoirMHHljTFNJAyWlS5rrcDkAAABVXmmz7pcuXaq33npLDRs2VI8ePRyoCoUqxPhJY0y4pI8khUt6zFpbrjsbGGPKWnJv66vaAAAAqqp27dqpS5cuOvfcc1WtWjVt2rTJ2xb0xhtveGfZwxmO/+0bY4IlfSipt6TPJL3obEVnz8fdSgAAAI64//779c033+ijjz5SamqqatWqpauuukqPPfaYevXq5XR5VZ6jQb4gxE+UNFTS55J+b0+jab+sXqSClfouPimynBg/CQAAKptx48Zp3LhxTpeBMjjWI2+MCZH0iaQbJX0s6WZrba5T9QAAAABu4siKvDEmTPkr8NdK+kDS7dZajxO1AAAAAG4U8BX5ggtbv1J+iH9HlTDE0yIPAADgXr4ez+4vPlmRN8YMkTSk4Nv6BY89jTETCr4+bK0dWfD1f5Q/K/6wpL2SnirlJlI/WWt/8kVtgUKLPADAbYwxstbK4/Gc1R06gcqmMMif6kanTvNVa00nSSfeJqtFwR9J2iWpMMg3L3iMk/TUSY75k49qAwAApQgPD1dmZqbS0tIUHR3tdDlAhZGWliYp/3ekIvNJkLfWjpU0tpz79vfFewIAgLMTHR2tzMxMHThwQJIUGRkpY0yFX4UE/MFaK2ut0tLSvL8TFf0E1/E58pWRW/qqAABVW2xsrNLS0pSenq6EhASnywEqlIiICMXGxjpdxkkR5H2E1QsAgNsEBQWpSZMmSkpKUkpKirKysliMQpVmjFF4eLiio6MVGxtb4a8dIcgDAFCFBQUFKS4uTnFxcU6XAuA0VezTDJdiLQMAAAD+RpD3ERprAAAAEEgEeQAAAMCFCPIAAACACxHk/YAL/gEAAOBvBHkfYfokAAAAAokgDwAAALgQQR4AAABwIYI8AAAA4EIEeR8xNMkDAAAggAjyAAAAgAsR5P3EMoMSAAAAfkSQBwAAAFyIIA8AAAC4EEEeAAAAcCGCvJ/QIg8AAAB/Isj7EBMoAQAAECgEeQAAAMCFCPIAAACACxHk/YQWeQAAAPgTQd6HaJEHAABAoBDkAQAAABciyAMAAAAuRJD3E8sgeQAAAPgRQd6HDIPkAQAAECAEeQAAAMCFCPJ+QmMNAAAA/Ikg70M01gAAACBQCPIAAACACxHkAQAAABciyPsJ0ycBAADgTwR5H2L6JAAAAAKFIA8AAAC4EEEeAAAAcCGCvJ9YJskDAADAjwjyPmSYJA8AAIAAIcgDAAAALkSQBwAAAFyIIO8nzJEHAACAPxHkfYkWeQAAAAQIQR4AAABwIYI8AAAA4EIEeR+iswYAAACBQpAHAAAAXIggDwAAALgQQd5PGD8JAAAAfyLI+5ChSR4AAAABQpAHAAAAXIggDwAAALiQT4K8MeYGY8xrxph5xpjjxhhrjJl4itf0MsZ8Y4xJMsakG2PWGGOGG2OCfVGT06xokgcAAID/hPjoOKMlnS8pVVKCpLYn29kYc62kyZIyJX0mKUnS1ZL+Jam3pKE+qiugDJPkAQAAECC+aq0ZIamNpBqS7jvZjsaYGpLelpQnqb+19k5r7aOSOklaKOkGY8yNPqoLAAAAqJR8EuSttbOttVusLdfQxRsk1ZH0qbV2WZFjZCp/ZV86xcmAGzB+EgAAAP7kxMWuAwoeZ5Syba6kdEm9jDHhgSvJNxg/CQAAgEDxVY/86Tin4HHziRustbnGmB2SzpXUQtKGkx3IGLO8jE0n7dEHAAAA3M6JFfmYgsfkMrYXPl8zALUAAAAAruTEivypFDaonLLL3FrbtdQD5K/Ud/FlUaeLFnkAAAD4kxMr8oUr7jFlbK9xwn6uQYs8AAAAAsWJIL+p4LHNiRuMMSGSmkvKlbQ9kEUBAAAAbuJEkP+x4HFQKdv6SoqQ9LO1NitwJQEAAADu4kSQnyTpsKQbjTHdCp80xlST9GzBt/92oC6fKt9IfQAAAODM+ORiV2PMEElDCr6tX/DY0xgzoeDrw9bakZJkrT1ujLlb+YH+J2PMp5KSJF2j/NGUkyR95ou6As0wSB4AAAAB4qupNZ0k3XbCcy0K/kjSLkkjCzdYa6cYY/pJGiXpeknVJG2V9LCkV8t5h1gAAACgyvJJkLfWjpU09jRfs0DSFb54fwAAAKCqcaJHvkrgIwUAAAD4E0Heh+iQBwAAQKAQ5AEAAAAXIsj7CZfrAgAAwJ8I8r5Ebw0AAAAChCAPAAAAuBBBHgAAAHAhgry/0CMPAAAAPyLI+xAt8gAAAAgUgjwAAADgQgR5AAAAwIUI8n5iaZIHAACAHxHkfcgYuuQBAAAQGAR5AAAAwIUI8n5i6awBAACAHxHkfYjOGgAAAAQKQR4AAABwIYI8AAAA4EIEeT+hRR4AAAD+RJD3IVrkAQAAECgEeQAAAMCFCPIAAACACxHk/cQySB4AAAB+RJD3IcMgeQAAAAQIQR4AAABwIYI8AAAA4EIEeT+hQx4AAAD+RJD3ITrkAQAAECgEeQAAAMCFCPJ+wvRJAAAA+BNB3oeYPgkAAIBAIcgDAAAALkSQBwAAAFyIIO8nlgGUAAAA8COCvE/RJA8AAIDAIMgDAAAALkSQBwAAAFyIIO8vtMgDAADAjwjyPsQceQAAAAQKQR4AAABwIYI8AAAA4EIEeT+hRR4AAAD+RJD3IVrkAQAAECgEeQAAAMCFCPJ+YumtAQAAgB8R5H2I8ZMAAAAIFII8AAAA4EIEeQAAAMCFCPJ+YhlACQAAAD8iyPuQYQAlAAAAAoQgDwAAALgQQR4AAABwIUeDvDHmSmPMLGNMgjEmwxiz3RjzhTGmp5N1+QJz5AEAAOBPjgV5Y8w/JE2T1EXSDEmvSFoh6VpJC4wxv3eqtjPFHHkAAAAESogTb2qMqS9ppKSDkjpaaw8V2XaxpB8l/U3SRCfqAwAAACo6p1bkmxa89+KiIV6SrLWzJaVIquNEYb5CZw0AAAD8yakgv0VStqTuxpi4ohuMMX0lRUv63onCzgadNQAAAAgUR1prrLVJxpjHJf1T0i/GmCmSjkhqKekaSd9J+uOpjmOMWV7Gpra+qhUAAACoiBwJ8pJkrX3ZGLNT0ruS7i6yaaukCSe23AAAAAD4lZNTax6TNEnSBOWvxEdK6ippu6SPjDHPn+oY1tqupf2RtNGPpZeLZf4kAAAA/MiRIG+M6S/pH5K+ttY+bK3dbq1Nt9aukPQbSXslPWKMaeFEfWfKMH8SAAAAAeLUivxVBY+zT9xgrU2XtET5tXUOZFEAAACAWzgV5MMLHssaMVn4fHYAagEAAABcx6kgP6/g8R5jTKOiG4wxgyX1lpQp6edAF+YrtMgDAADAn5yaWjNJ+XPiL5G0wRjzlaQDktopv+3GSHrCWnvEofoAAACACs2pOfIeY8wVkv4k6UblX+AaISlJ0jeSXrXWznKiNgAAAMANnJwjnyPp5YI/AAAAAE6DY3PkAQAAAJw5grwPMUYeAAAAgUKQBwAAAFyIIO8njJ8EAACAPxHkfSjhaIb3aw9JHgAAAH5EkPeTL1ckOF0CAAAAKjGCvJ9sOpjidAkAAACoxAjyfjJz/UGnSwAAAEAlRpAHAAAAXIggDwAAALgQQR4AAABwIYI8AAAA4EIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIggDwAAALgQQR4AAABwIYI8AAAA4EIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIggDwAAALgQQR4AAABwIYI8AAAA4EIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIggDwAAALgQQR4AAABwIYI8AAAA4EIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIggDwAAALgQQd5PWtaJdLoEAAAAVGIEeT8JCwl2ugQAAABUYgR5P7HWOl0CAAAAKjGCPAAAAOBCBHkAAADAhQjyAAAAgAsR5AEAAAAXIsgDAAAALkSQBwAAAFyIIA8AAAC4kONB3hjTxxgz2Riz3xiTVfA4yxhzhdO1AQAAABVViJNvbowZLekZSYclTZO0X1KcpM6S+kv6xrHiAAAAgArMsSBvjBmq/BD/vaTrrLUpJ2wPdaQwAAAAwAUcaa0xxgRJ+oekdEk3nxjiJclamxPwwgAAAACXcGpFvpek5pImSTpqjLlSUgdJmZKWWGsXOlSXz1jrdAUAAACozJwK8hcUPB6UtELSeUU3GmPmSrrBWpt4soMYY5aXsantWVcIAAAAVGBOTa2pW/B4r6Tqki6RFK38VfmZkvpK+sKZ0nzDGKcrAAAAQGXm1Ip8cMGjUf7K++qC79cbY34jabOkfsaYnidrs7HWdi3t+YKV+i6+LPh00VoDAAAAf3JqRf5oweP2IiFekmStzVD+qrwkdQ9oVQAAAIBLOBXkNxU8Hitje2HQrx6AWgAAAADXcSrIz5WUK6m1MSaslO0dCh53BqwiAAAAwEUcCfLW2sOSPpMUI+mpotuMMZdKulxSsqQZga/ON9o1iHa6BAAAAFRijt3ZVdLDknpIGmWM6StpiaSmkn4jKU/S3dbaslpvKqRzG9bQ+n3HJUnnNa7pcDUAAACozJxqrZG19pDyg/y/JDWR9KCkAZKmS+pjrXXd+MnuzWO9X1vG1gAAAMCPnFyRl7U2Sfkr8w87WYevGDE8HgAAAIHh2Io8AAAAgDNHkAcAAABciCAPAAAAuBBB3ocMLfIAAAAIEII8AAAA4EIEeQAAAMCFCPIAAACACxHk/YT7QQEAAMCfCPI+xLWuAAAACBSCPAAAAOBCBHkAAADAhQjyfmJFkzwAAAD8hyDvQ9wQCgAAAIFCkAcAAABciCAPAAAAuBBB3k+YIw8AAAB/Isj7kKFJHgAAAAFCkAcAAABciCAPAAAAuBBBHgAAAHAhgryfcK0rAAAA/Ikg70OpWbnerw8dz3KwEgAAAFR2BHkf+njxbu/X7y7Y4WAlAAAAqOwI8gAAAIALEeQBAAAAFyLIAwAAAC5EkAcAAABciCAPAAAAuBBBHgAAAHAhgrwPnduwhvfrBjHVHKwEAAAAlR1B3od+07mR9+tL2tVzsBIAAABUdgR5HwoJMt6vjTnJjgAAAMBZIsj7kCmS3q11sBAAAABUegR5Hyq6Cn/weKZzhQAAAKDSI8j7UEZ2nvfrWb8cdLASAAAAVHYEeR9aujPJ6RIAAABQRRDkfYorXAEAABAYBHkAAADAhQjyPsTISQAAAAQKQd6HyPEAAAAIFIK8D7EiDwAAgEAhyPuQYU0eAAAAAUKQBwAAAFyIIO9DQfxtAgAAIECInj5Eaw0AAAAChSAPAAAAuBBBHgAAAHAhgrwPLdp+xOkSAAAAUEUQ5H3oSFq20yUAAACgiiDIAwAAAC5EkAcAAABcqMIEeWPMMGOMLfhzl9P1AAAAABVZhQjyxpgmkl6TlOp0LQAAAIAbOB7kjTFG0nuSjkj6j8PlAAAAAK7geJCX9KCkAZJul5TmcC0AAACAKzga5I0x7SSNl/SKtXauk7UAAAAAbuJYkDfGhEj6UNJuSX9xqg5f6t2qttMlAAAAoIoIcfC9n5LUWdJF1tqMMzmAMWZ5GZvannFVZ+GcejW0YCt3dwUAAID/ObIib4zprvxV+JestQudqMEfjHG6AgAAAFQVAV+RL9JSs1nSmLM5lrW2axnvsVxSl7M59pkIDiLJAwAAIDCcWJGPktRGUjtJmUVuAmUlPV2wz9sFz73sQH1njBgPAACAQHGiRz5L0jtlbOui/L75+ZI2SXJV200QK/IAAAAIkIAH+YILW+8qbZsxZqzyg/z71tr/BrIuXwimSR4AAAABUhFuCFVpkOMBAAAQKAR5HzIkeQAAAARIhQry1tqx1lrjxrYaSdp/7IzG4QMAAACnrUIFeQAAAADlQ5D3oUva13O6BAAAAFQRBHkfiqke6nQJAAAAqCII8gAAAIALEeQBAAAAFyLI+xDDJwEAABAoBHkAAADAhQjyAAAAgAsR5AEAAAAXIsj7UOPYCKdLAAAAQBVBkPehutHhTpcAAACAKoIg70NFp9YEMcIGAAAAfkSQ9yFjfk3vHutgIQAAAKj0CPI+dOIi/JqEY47UAQAAgMqPIO9D5oQk/9OmRGcKAQAAQKVHkPchc2KSBwAAAPyEIO9HaVm5TpcAAACASoog70fbEtOcLgEAAACVFEHejzJz8pwuAQAAAJUUQd6PsvM8TpcAAACASoog70cehskDAADATwjyfnRB81inSwAAAEAlRZD3o5Z1opwuAQAAAJUUQd6PrKW1BgAAAP5BkPejzFwudgUAAIB/EOT9aO/RDKdLAAAAQCVFkPejdg2inS4BAAAAlRRB3sfaN6jhdAkAAACoAgjyPta63q+Tajxc7AoAAAA/Icj7WLAx3q+5sSsAAAD8hSDvY0FBRYM8SR4AAAD+QZD3sW2Jqd6vJ6/Y62AlAAAAqMwI8j62cvcx79dLdiQ5WAkAAAAqM4I8AAAA4EIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIgg72eZOXlOlwAAAIBKiCDvZ7e+u8TpEgAAAFAJEeT9jJtCAQAAwB8I8gAAAIALEeQBAAAAFyLI+1j35rFOlwAAAIAqgCDvY7UiQp0uAQAAAFUAQR4AAABwIYI8AAAA4EIE+QBYsfuo0yUAAADEz/pqAAAgAElEQVSgkiHIB0BSarbTJQAAAKCSIcj7mLUlnwvibxkAAAA+RsT0MWNKPrdqT3LgCwEAAECl5kiQN8bUNsbcZYz5yhiz1RiTYYxJNsbMN8bcaYxx7QlGRFhIiede/WGLA5UAAACgMiuZOgNjqKR/S9ovabak3ZLqSbpO0n8lDTbGDLW2tEYVd/J4rIKCSlmuBwAAAM6AUyvfmyVdI6mxtfYWa+2T1to7JLWVtEfS9coP9a5T1rlHj3E/aPH2IwGuBgAAAJWVI0HeWvujtfZ/1lrPCc8fkPSfgm/7B7wwHyjrI4TElCz97q1FAa0FAAAAlVdF7EXPKXjMdbSKM+SpNM1AAAAAqMic6pEvlTEmRNKtBd/OKMf+y8vY1NZnRZ2m8xrV0P9W73Pq7QEAAFBFVLQV+fGSOkj6xlo70+lizsQfejV3ugQAAABUARVmRd4Y86CkRyRtlDSsPK+x1nYt41jLJXXxXXXlFxZS0c6NAAAAUBlViNRpjPmTpFck/SLpYmttksMlnZVbezYtc1tyRk6Z2wAAAIDycjzIG2OGS3pd0jrlh/gDDpd01u66qEWZ28Z/uyGAlQAAAKCycjTIG2Mel/QvSauUH+IPOVmPr5ysvebnbcySBwAAwNlzLMgbY8Yo/+LW5ZIGWmsPO1WLr9WPqVbmtspzr1oAAAA4yZGLXY0xt0n6m6Q8SfMkPWiMOXG3ndbaCQEuze9smbeMAgAAAMrPqak1hTMagyUNL2OfOZImBKSaACrPivzyXUe1NuGYftOlsWKqh/q/KAAAALiOI0HeWjtW0lgn3ttp1kqZOXlaujNJ3ZrGqnpYcLHtiSlZuv7fP0uS1u87rheGnu9EmQAAAKjgHJ9aU9VYa3X3B8s07J0l+sN7S0ps/2L5niJfJwSyNAAAALhIhbkhVFWxLzlT+5IzJUmLdyQpO9fjnXKz5WCKnp+xyS/vm5PnUUiQUSnXIgAAAMCFWJF32B8/XCZb0Dh/67slV+glaVtiqsZ9s0FLd57ZfbJW7j6qnuN+0KCX5yk1K/eMawUAAEDFQZD3k3v7tSzXfrM3JeqDhbs0YcEO7S9YqS9xrA+X68252zX0PwuVnv1rEE/PztW4bzfohZkblZmTV+Z7DHtniQ6nZmvTwRT9c9bm0/tBAAAAUCHRWuMnQafRwfL01+vL3Hbvh8u15VCq9/vNB1N1bsMa+nHjIf3xw+Xe5yPDQ3R//1be7/M8VqOnrNPeYxnFVuHfXbBDT13dvtT3yszJ00+bDqlzfC3Vq1H2LHwAAAA4jyDvJ7GRYT45zoz1B4p9P+SNBTKm5BjLV3/YUizIf7p0tz5ZsrvUY6Zl5SoyvOT/9E9PXa/Plu1RnehwfT+in0JDjCLCyv4nkpvnUfBZ9t2vTUjW5BUJuvr8huratNYZHwcAAKCqIcj7ye8vbKpnp2/wy7FLm0WfmeNRuzEzFBkerC/v663vfjlY5uuzcz2KDC/5/GfL8ifmJKZk6fy/zVJ0tRB9dX9vtaob5d0nz2MVHGT0y77jumPCUkWEBevS9vUUEmx0X/9WiirlBOFkrn59viRpws87te3vVyj4dD7KAAAAqMII8n5SLTT41Dv5WEZOnjJy8tT3hdlqXKt6mft5rNVv31yoJTuS9O1DfdSuQY1S90vJzNWwdxbrn7/tpB7NY7Vi91HdO3GFGteqroSj6Tqcmi1JenPudklSrsfqycHtih1jW2Kqdh9JV982dU4Z0qet2acrzmug0GAu3QAAADgVEpMfPTSwtWPvnXA0o8xtXZ/9Xkt25E/AGfzKPO/UnNLsT87UTW8v0guzNumG/yzU4dQsrdpzzBvii3pzznZ9umS398Lbg8czdfm/5ur2CUv1zvztys71FNv/yxXF5+Q/9OkqvTCz+PjNw6lZ+nr1Ph3PzCnxftZaLdmRpGU7k076M7jFweOZmrFu/0kvXAYAAChkKkMAOpExZnmXLl26LF++/NQ7+1FmTp7ajpnhaA3lcV3nRqoTHe5dWT9bTwxuq3v6tNCd7y/V7E2JJbY3rlVd9/ZrqdFT1pX6+p3jr5SUH9Qv/ddcbT2UqgFt6+rdP1xQbL85mxN1W8HIzo/u6qHereLOqu69xzL0+o9b1a5BtG7t2axcr9l8MEXvzt+h/ufUVb82dRQUJIWHnP6nMdm5HvX+x49KTMnS77o10T9u6HjaxzgduXkehfDJBwAAZ61r165asWLFCmtt10C/N0Hez46mZavzM985XYarLHhigA4ez5THY3XDfxZ6n985/kpl5uTpnfk71CIuUg98slJ5nvx/vxFhwfrlb4PK/R6r9xzTR4t36aqODdW3TR1Za/W7NxdpScGs/kn39lS3ZrGnPE7XZ77TkbTin070bFFbn9xzoaT8oB8fG3HKVqsfNhzUne8vK/aznsw3a/fr3fk7dHOPeF3XpfEp6yxq1FdrNXlFgh4f1Fa3925+Wq89W4X/veHGZACAysLJIE+PvJ/V8tH0mqqk9/gfS33eWlvmJxzp2Xm6b+JyDencSJe2q6fjmTm68tX5alSzuv4zrGuJKUJD31yo7FyPPl+WUOrxbvjPQjWPi9S/f99FbevX0J6kdE1fu1+Xtq+nlnXyL/79dMnuEiFekhZuP6JlO5OKnYRsfW6wdwU8J8+jtXuTtetImqyV4qLC9eXKvaf+iyni/o9WSJKW7Tqqqzo2VFhIkPYey9AXy/aoT+s6ZU4A2nssQx8tzp9m9Nf//RLQIH8oJVO3vrNEOXkevfeH7oqvHRGw9wYAoDIiyMM1vlheeugu9O26A/p2XfFxnXuPZajX+B+08ZnBOpaerWPpOaofU61Ev35pdhxO06CX52lA27r6ceMhSdL4bzeq/zl19OcBrfXEl2vLfG3REC/ljxG9qmNDSdJt7y7Rz9uOnPS9F2w9rO2JqdqWmKZf9h3XC0M7qmntSE1anqCpq4qH/uw8j8JCgnT/Ryu0es8xvfz9Fq3/6+WljhhNTi9+rcGSHUnq3rz4Jw/ZufnHO9H2xFTViQ5XdLXQk9ZemnV7k3XVa/O934/4fJUm39frtI8DAAB+RWtNALw4c5Nen73V6TLgoGqhQbq1ZzMNaFtXN7616IyO8e1DfTT4lXklnr/m/IZqXKu6/u+nbd7nakeG6a4+LZSckaOcPI/u799StaPC9cu+47ri1eLHGHRufd3Zp7naN6ihy/41V3uPZej+/i312KC2kqRj6dn67peDenTSGknS/f1b6u4+LbyfNllrtX7fccXXjlCNUkL+az9s0Uvflbyj8Knah3xhe2KqVu4+pss71D/t0ahFLd5+RD9uPKTfXdBELepEnfoFAIAqgx55H6toQX5NwjFd8/oCp8tAFbf4LwP1zdr9+uv/fil1+40XNNGnS/d4v9/y3GBNXLSrzP0XPTlQUdVC9Mr3m/X2vB2SpDVjLysW5ncfSVffF2aX+vryBvmlO5M0Zso6dWlaS88N6VDu/vqM7Dz1+Pv3Op6ZqxsvaKLx15/ZBcRpWbk69+mZkqT42Aj98Eg/bTqQovYNaijIj/c9sNbq8clrtHhHkp4d0kF9Wtfx23v5QkpmjnLyrGIjw7T3WIZqR4Y5MoYXAAKNHvlKriUreKgAevz9h5NuLxriJan1qG9Puv+9E5dry8EUpWX/Oi6z49hZevl3nTSkcyNJ0rsLdpSrttV7jmnnkTQN6lC/xNSfoQVtShsPpOjSdvV0cdu63m1fr96nScsTdEfvZup/Tt1ir5v1ywEdz8z1/mxFg7y1tsQJQcLRdDWMqa6MgguqY6qHatiFTbX5YIp3n91J6brlv4u941tv6h6vv//m9E4u/vTxCiWlZevl33VSs7jIUvfL81jNXH/Aew3HsHeWFDvxKc/UocOpWdp1JF3nNYoptVXKl3YfSdfgV+Yqx2N1fZdG+mTJHtWNDtecRy9W9TD3hPnUrFwlZ+SoUc2y78MBABUJQT4AIsND9MndF+qmt8+spQKoiFbtOVbq88M/W6VuzWpp6c4kTfh5Z5mvn73pkMJDghQfG6Fr38j/xKpOdLiWjrpEx9KzNWXlXm05lFrsNSM+X6X/PXCRmsRGKDUrVw9+slKSNHdzonaOv1JZuXmavma/GtasrsSUrGKv7fP8j3rvD93106ZDenPudvVvU0ejrmynmOqhGj9jo96cs109W9RW5/ia3jalp79erzsvKn5BcGGIl6RPluzWxefU0WXn1i/x8yWmZGnP0XR1blJTxhilZuXqtR+3eK+3GPDST9o+ruSnEunZuRr08jztTkov9e/tsUmrNW3Nfv3linb6/YVNdTg1S8/P2KjYyHCNuLS1wkOClZiSpQue+977moVPDlCDmLLD6ZSVe/XVyr26p28LBQcZvfrDFt3cI957XUdmTp7CQ4LKPGF5dNJq7wndJ0vyTwgPpWTpv/O2688BuJ9Gdq5HuR6PIsLO/P/SktKy1e+F2UrNytW/b+miQR0a+LDCyiMrN09hwWX/WwAQWLTWBNCTX67VJ0t2O10GUKG9fWs33f3BspPuM6RTQ21NTNW6vccDVNWp9W5VW2/c3EUx1UM1f+thDXsn/x4Hjw9qq05NauqOCUuVccLNvl4cer5u6Fp8fOgLMzfqjdnbdKKd46/U1kOpuuSfc05axzXnN1S10KBiE5n6tamj127u7G17SsnM8V60fDwzRx3Hzir1WGvHXqY1Ccm6d+JypWTmqm+bOrq9V7Nin4pIUs9xP2h/cmaJ1/+hVzONvebck9Z7Kh5PfovR5oMpeu4356lDo5hi2w8kZ+qq1+YrMydPH93VQ+c3qVnuYxd+MpOalatn/veLPlv266dShSeGy3cdVZf4WmfUJlTaJz+lWb3nmHLyPOratNZpB+Q9Sel6b8FO9WgRq8tLOaH0pbmbE/Wnj1eoae0ITb6vlysD/Y7DaRr+6UrVigzTv2/p6qpPjAIlJ8+jZTuPqlOTmvz9lBM98j5WcYP8Gu9qFQBIUrsGNXToeKaOpGWrTb0oHU3PKfFpwumqX6OaDhwvGawb1ayuvcfy7/r85wGt9Mhl55z05CA2MkxJpYxY/fmJAWpYpP2krCB/e+9mevrq/CCfk+fR9DX7FRkeokva1S13APxyRYIe/ny1JCnIqMSnGPd+uFwz1udPqwoNNlo79vJSQ/eJ7UjT1+zX01+v1+HULIUEGeV6iv9/4TXnN9TXq/dJknq1rK2P775Qh1OzVDsy7JS1Z2Tn6blvftHERbt1fuMYTbyrR4lpT7l5Hs1cf1CbDqbo1R+2SJLe/UM3DWhbr9TaM3LyVD00uNjPYK1V8ye/8X6/4IkB5W4L+nbtfs3dclh3XtRcrer+2v75fz9t1ardxzTy8nPUpl50sdc0e2K69+vakWHKyfPotl7N9Mhl55TrPU9l5+E0hYUEFfu35WuDX5mnDfvzFwCKXtRf0f287bD+t3q/bureRB0bl/9k9Uz86eMVmr5mvzo2jtHUP/V23cmaE+iRryIq4TkTgLNUGCokafPB1JPsWX6lhXhJ3hAvSa/9uFUx1UNLXFtQVGkhXpJ6jf9RUeEhuvK8Bhp7zbmlhnhJmrMpUdOb7lefNnGasfaAHpucP/noo7t6qGeL2np+5iYlHE1Xq7pRevn7Ld7X3d67meKiwhUXFaa5Ww57n/fYkqvchSFeknLyrLo/970eGNBKf/9mY4l6HhzYWg9f2kZSflgpdGKIl+QN8ZL087Yj+t2bC7V4R5J6t6qtiXf2kDFGaVm5Cg0OKnENwtvztmviovxPX1cnJOvFmZv012s7FNtnyqp9GvnF6mLP3TFhmXaOv1LWWr0wc1OxSVSF/n1LFw0+L7/t55u1xcftztucqBu7x8taq6xcT5knNAu3H9F9BfeimLs5UQueGKAlO5L05pxt+qGg9Wvd3mT9/OTAEq8vVHgPjdd+3Kr7+rcsV1vT0p1J2nYoVdd0alhi/5+3HtbN/12s4CCj6Q9epLb1a5R4/dG0bEVVC1FowclMalausnM93vuElPVzZ+bkeZ8r+vs2d0uiHhvUVh8s3KllO4/qwYGt1apuVLk/SSl8z/Lsezg1S+8t2KG29Wvo6vMblrlf0VoLZeXm6ea3F0vKb+fz98Sv6Wv2S5LWJCTr69X7tD85U0O7NlbtqPCzPnZunkfztx5W+4Y1VDe62lkfD6zIB9Tjk9aU+Oi26AoHAFQVz/2mg0Z9te6MX9+xcYzu799S905cceqdi4iPjdCIS1trxGerT71zGa7r3Eg/bzviPWG6sEWsrjivgUKDg/RkGfeXuLdfS93as6kSjmbogma1iq2kF9Uwppr2lXFiVOiVGztpUIf6Omd0yRvkdWhUw9tydlP3eD1z7bkKCQ5Sbp5HD366skT4P5mXhp6vmhGh6tSkptKz89Tn+dInUC35y0DlWeu9DqNouM3J82jGugPKyM7znsj96eKWGnFJm2KfLjR/crp3sSs02GjTM4OLTYX6cNEujZmyzvv6VnWjNGbKemXnejThjguUmZOnv3+zUQeSM/XazZ11ccEJ6hOT12jS8gTVj6mm33VrUmIU7qf3XOgdCdyoZnUFBUl7kjL0/PUdNbRbY+V5bJkXln+2dLfGfbtRgzs00Ljrzit1n91H0vXezzv04cJd3hPGaX++yNsilpKZozmbEzVj3QFNKwjQkjT5vp7q2jT/Hh8HkjN14bhfhxUUBvmUzBwFB5kyT6IysvM04rNVOpaRrReHnq9aEWH6aVOiMnPy9MHCnZIxemtYV6Vk5mjelsO6smMD1Y2uVmouGXRuff1nWPkWm7ceSlV4SJCaxJa86d9z03/R2/N2qGZEqBY9ObBcLWsHkjNVMyJU1UKDZa3Vsl1HVT00uESbnZNorfGxihrkH5u0uljf6s7xV2ru5kR9smS31u1L1p6k/NWy6Q9epCtfnV/WYQAAVdyVHRt4V05P5ecnBujHjYc0esqZnzg1j4vUjsNpJ93nhRs6qmZEmMZMWac+reP0wtDz9fHi3frLV6Wf3NzUPV6/7EtWTESY5m5OLLG9e7NYXXZuPXVqUrPETfZOZef4K8969HObelFKSsvWff1baWDbuvp69T51a1ZL5zaM0RfL9ujZ6Ru8+7atH61vH+pTYnV+0MtztfFAyomHliTVjQ7XoZO00RUG9qmr9uqhT1d5n7+gWS1d26mR93/P/97aTZe0z2/J8nis5m5JVFR4iF6ctUmLtieVPHARYSFBCgky+SdqrePUoVGM/l3KJ0FS/khiI5V6YpOZk6c5mxOVlpWrhz9fLWOk6X/uo/YNa8jjsVqVcEzt6tdQu6d+Pfkcf915urF7vGZvOqQjqdm6+vwGysmzSsvKVY1qodqXnKEN+4/rwU9WKjYyTG8O66bHJ6/R1oIhCF/d30ud40u/i3mgEeR9rKIG+Ue/WF3s7qRFPx7zeKzW7k3WOfWjVS00mJV6AADOUEz1UCVn5Jx6xzMQHR6ilKzcUrfddVFztW9YQy9/v0WXtKtX7hG8pbm5R7yGX9Ja3Z87+ehgKf/n/fahPlq848hZfdpUHi3qROqbB/soMydPNSPy25oe/nyVvlyxt8S+H9/dQ9+s3e9tNTuZ/ufU0U+bSp7QlSUuKlxLRw3Us9M36J35+av8k+7tVeyaj0AhyPtYRQ3yI79YrUllBPkTEeQBAKjaIsKClZ6dd+od4TVzeF+dUz/61Dv6kJNB3r93CUExp3MTyEvbl5xcUNTzN3TU6qcuU+u6UaoWGqR2DWqobf1o1YkO1yXt6mlIp9Ivpvljvxblev9ZI/qWuW3Ts4P0/cP9ynUcAABwZgjxp+++iRVrEdffmFoTQMMvaaNJyxPksdJjg04+ruu5IR303S8HJUkhQUbdmtUq1utWKyJMMRGhmjWib6kX44w6oSdx9JXtdHvv5goy+TOlY6qH6pu1+7Vxf4ru6dtCP287og8W7lTLOlF6cej5ahYXWerouW8f6qPwkGC1qhuluKhwHU4tvb/vgma19JvOjfWXr9aqYUw1/Tiyf5ktQxv+Nkhp2bnq9uz3pRzp1BrXqq6Eoxmn3hEAAFRq209xLUdlQ2tNgK3bm6ydR9J0Wfv65bpt+u4j6YqNClNkWHCxKQcT7+yhi1rHlfm6xJQs9R7/o7LzPHrqqva644S7U5ZH4TiwQmvGXua9oYyUP/przJR1urBFrK7r0lgea7X5QKrq1ghX3ehwGWOUcDRddaLDFR6Sf2X6jHX7vVMm/tivhYYPbKPqYcHK81i1/EvpUxyk/Au77u3bUu0b1lDC0XTd/t5ShYUE6YM7u+vjxbuLja4DAABVl79HdJ6IHnkfq8hB/mw8P2Oj/u+nbWpTL0ozHupbbDRXafYey1BCUrq6N4894xs6TFy0SzsOp+nefi1VJ/rsZ8ieTNGfb/z1HXXd//0sSWoQU00zR/QtdhJR+O/WGKPNB1N02b/merd9eGd3TVm5T5NXJOhUzm9SUy8NPV+t6kbpSGqWup7hpwKncm7DGlq/r+LchRQAgMqKIO9ylTXIW2u15VCqmtWOLNdqvtvtPZYhI5XrLn9TV+3Vqj3HdHefFmpYs7rSs3M14MU5OpSSqVLu9aIfHumnFnGRJU5wpq3Zpwc+XnnK96sZEaqHL22jp6au9z5X1iSDt4Z11WUFt04f9s5izStygxsAAOBbVSnI0yPvIsaYErfMrszKe6txSbq2UyNd26mR9/uIsBDNeay/ktNz9NjkNcVGWv3p4pZqWaf08VRXdWyo1MxcPVHGTV0eG3SOftetiWILbtPet3UdbTqYoovPqatdR9I0Z3NisdnCEWHB3hAvSR/e2UO5eR5Z5c8X3pb4ay/fny5uqb6t6+h3BTcnkfJbqH7/zq/tTbNG9C326cOE2y+QlXT7e0vL/Lu5pUe8HhjQSrGRYcVuIFMnOlyJZcwwvrJjA93Ru5k+XLhLU1b9eofLk41dO5U7ejc/q1FsAACgOII8Kq3wkGDVrRGs8dd11KOTVmvelsP6Y78Wur9/q5O+7sbu8RrQtq62HkrV16v3aWi3xgoOCtKuI2ka1KG+t99fkprFRapZXKQkqXW9aLWuF10syBdtBypUeGHyrBH9il0X0CW+Von2pYtax+nTey7Uv77brIHt6qpNvWhdfX5D/W/1PnVvHqv+BXcvXP3UZUpMzVJYcJDia0do44Hjuu7/flavlrX112vO9b7nB3d0110fLFPN6qH6bkQ/Xf36fO1OSve+399/c576nVPHexLVJb5WsSD/7fA+6vv8bO+nHP974CIt35Wkt+Zu18094nV77+b6aPEuzVx/UMt3HfW+7pkhHTTswqZ68oq2uvb1Bfpl/8nbjIKDjF6/qbMubltXs345qB83HCxWBwAAoLUG8Lmik3kaxFTTwicHlrnvp0t2a8zUdeoSX0uf3nOhjDH67ZsLtWRHkm7r2VR/vbZDidfkFdw8rH2DGmfUYpWcnqPqYcEKCwnSDxsO6s73l0mSRl3RTnf3LTmedPeRdH21cq+u79pIjWtFaMfhNO1PzlDPFrVPeu3F1kMp2nQgVZe0r1vs5Mdaq7TsPFUPDdbcLYneTxPCgoO0ZNRAVQ8LLrZ/oQ37j+vaNxYoO9dT7PlqoUHKzPGU2P9EhR+15uR51HrUt6fc/2RCg41y8kr+tzM+NkK1IkK1OiH5rI4PADgzQUbaPq7qtNYQ5AEfu/7fP3tXo2/qHq9x15130v1Ts3IVGRbsDcV5HqudR9JK7eH3NWutpq7ap5SsXA3t2ljVQksGaH9LyczR/1bvV8fGMerQKOak+x5Lz5a1UudnvvM+t/qpy3Q0PVv9X/yp2L6RYcFKKzKDuWjP5Pe/HNRdH+SfwPRuVVsLth4p8z2XjBqogS/NUUpmroZf0lrDL2kjqeRN2/7Yr4WeHNxOkpSenavUzFwlHMvwXrRdaPqDF2n0lHVaufuY97mP7uqhW4pMiNox7gr987vNeu3HrWXWdeLJRHCQUV4pF4Rc36XxSS/8/us15+r5GRuL/V2djY3PDNJd7y/T/K1cCwIg8GIjw7RizKUBfU965IFK5F+/7aQ/fbxCEWHBemJw21PuHxVe/NcwOMiU2cPva8YYDenc6NQ7+lF0tVDd3CO+XPsW3g78v7d200eLd+mWHk0VExGqmIhQhQQZ5RYE2Tt6N9fDl7VRv+dn60hatl4cen6x41zSvp52jr9S1loZY7wXkt/+/+3deXxU1d3H8c+PQCBsYQuKEJRNBEFZxAUUARWV4lKrrVas2tVHW9pan1p9VYt9WrWW9nFt3Ze6tWKt1qdWqcqmqFh3LIVCiOxElkA2EpKc549zEibDTDaSmUzyfb9e87rJvefMvfObc+f+7noeeZcuHdPo1CGN5Rt38bOzjqRvt058Muf0Wpfr3lnjOWPUvnshOqe3p3N6e/p278Qvzh3FT59fDvg+IY48JJMvju1fnchn98pg0tA+nDqiL6+uyOPeWeMwM340fThjB/bgibfXceGEbPIKSqvf56Rhffj9rPF87aF3eH9dPuMG9uC5KydVP81p9Jz5FIZ7GW48ayT9Mjtx94L9dwouPm4gl048jHPH9OfTTbs4dlAv2qe148P1+Vz2yDKKSstZ+pNT6JyexlE3zaei0nHB+AHMOftI/vbxZkYe0p2KSsc597wJwGUTD6NThzQe/8axjLjx5ZhnSmYdP3C/7tovOjab6UcezLaCUs4ecwgrNhewvbC0+mxRffXp2pH7LhnHix9t5tGluQ2qGy1WPxpVxg3sQVlFJcs3xr9E7ITBvXkrZ98O4sh+3eu8pExEDtyIfm3nXkLQEXkRaSVW5xXy4JIcJg3tw1lH+56NS8oq2FZYSnavzvV6j4pKRzvzOziFpeX77WRF+ssHG7j+ueWcMKQ3D116TK1nT97O2c6LH23iwgkDGT0gk/KKSq6Z9xGf7Sjm1vOOqld34mXllTpQGtIAABViSURBVLz0yWZ6dknn5MOzAKisdHy2o5jDeneuMf9VWwv4w1u5nDrioOr7KDbsLKZXl3Q+WJfPDS8sJ6trR+7/2jFkZux/HwfAnr0VpKe1q37M7dbde1i5pYBJQ/uQFvXo2/9sLeCTjbs4Y9TBdE73MdtZVMay3B1MHpbFms8LeeiNtcw+ZRhdOqYx8ZbXKa90fOWYbE4beRAnD8+iQ9r+l4ndu2gNH6zbybcnD+bTTbvp3aUjVz31fvX0wVldKK9w1fd5TDuiLw9fNqF6/sff8hql5ZVcfNxAzhvXn2ff28DTy9bH/LxzzhrJnBf/Bfibve/56riYj6T979OHc9VUf5/NjqIy0tu3Y+WW3Xzp92/VKJd76xe49OFlLFrlb7S/9owjyC8p475FOQCcdfQhlJRV8OqKrfs+T58uNTqzuWa6P/szd/6q6nHxblJfe8sMnl62nttfXUVe1PTZpwyjuLScS044lB4Z6dy/ZA3llY5VWwpYEB4E0LVje24+bzSzn/ZP7aqto71DMjux9LpTqh8Z3BBjsnswJKsrL3y4sXrHu6G6d2rP7j2Nu+m+ud110Vi+93TdTz5LFV85Jps//TP2OtNS/fHbx3P84N4JnacurWliSuRFJBHKyivbxKNgm9rqvELWbiti6vCs/Xqlrs3uPXs5as786v/fuf4UdhaXcfED75Devh3PXTmRfpn7nna1bnsxqz8vYPKwffPZlF/CY0t9L9Y/++unlFVU8vBlE6p3jnYV7yWz876dm6pHxk4ZnsWVU4Yy4bCeMXfaPt6Qz9l3+zMT35s2lB9NH87W3Xu48YXlZGZ04H/OHcXeCsf9i3Po3qk9l08aVL1DVFJWwcb8Yob27UZewR6ee38jxw/uzZjsHuzZW8Hk2xaQV1DK3AuO5vzxAyivqGTttiJOi3iCVeSlY5GXfZ0z5hDuuHBszHgW7NnLK59u5ZhDe1bftP/R+nzeXLONrxyTzf2Lc7h/SQ4XHzeQr08axNl3v0l5ZSXzvjOR0QMyKS4r54zbl7BuRzE9Oncgv3gv4HdIyitdjRvpwV9WduQhmTXmv3nXHs4NZ3ROGXEQi1bmUVxWUSPJf/Xqyby8fAtz56+if48MFlwzhYseeLvGDfUAh/buzG8uOJrz7625U1WbtbfMYN4/N/DjP39cY/y1ZxzBdyYPZu32Iu5ZsJo+XTsyY3S/6mUFOHPUwfx+1niO/eWr1TtPb/5kGjPvXMLOEIvrzjyCorIKenXuUL2jGO3q0w7nt/9YVWNcevt2jBnQg2W5O2qMX3vLDI66aT4FMXZkrpwyhL9+tIlDe3fm4uMO5dGluSxbW7P+1OFZ1TtvdXnhqkkcnd2D429+jS2799SrDsDPzzmSsdk9GdK3C+f9bimfbS9myvAs/r58S631rpl+eI2d1sZacM0UBoX2nChK5JuYEnkRkdbpibc/Y957G/ivkwdzxqh+AJSWV5Bm1qCdAvA7BsWlFRyc2SlumYpKx5rPCxnWt2ud96y899kOVucVcvbR/clIb7r7TZxzlFe6/c5aXPfcJ7z0yWZ++oURXHBMdvX411Zs5b7FOZw/bgBfnpAd/XYNEnlmqrC0nIpKV+MsTmWlo3hvBV07tufd3B28vWY7X56QzY6iMi5+8B06pBn3zhpP/54Z9O0WO84Fe/bi8E/5qroZftrchWwvKuOOC8cw8yh/hm3llgIG9upMRnpadTnnHLe9vJL2acaPTz+CjPQ0yisqueiBt3k3dycnDu3DG6u3YQaLrpnKgJ4ZPLDEnxW58NiBZGb4ef76lZWs3FLAxKF9GJLVhZMPz4r5fW/ML+GqJ98no0Ma984aT2bnDqzcUsD9i3OYfHgfzhnTn3Xbi3l0aS4Th/Tm1JEH1YiVw18++Y1H3+W1f+dx1dQh/PfpR7B+RzEn3bYAgN9dPI4Zo33bvuLx93j5030JcNUlgRt2ljDjziXVCf3saUO5evrwmPF96p11zHtvPd8+aTBnju5HUWk5Y3/+Dzq2b8edF43l8kf9AwdmHtWPG2aO5N5FaxhxcPfqtvOvTbv52sPL2FZY80zPeeP689z7G2uM69O1I29cO3W/+62cc3ywPp9dxXur5xct5+YZPLlsHU+9s44rTh7MkKyuzLzrDcA/SKBLx/acN7Y/Xz9xEGu3FXHqbxft9x4zRh/M7y5OeC6tRL6pKZEXEZG2oLLS1dnLd7I0dgerqu6ukr1xk//6Kq+oZNGqzzmkRwYj+nU/oPdqSs45thWW1dlj+qb8Eqb8eiFlFZXc/dWx1Ts14C+he/+znUw7oi99uzc8TlVt552c7Xy4Pp8LQh8pseytqCQt7Ng8+EYOhXvK+dbkwTywOIc7X19N327+Ur2hfbvWekkiwNPL1rFySwHHDerFD5/5kA5p7Xhp9kkxL4HM3VZEaXllzMsPS8srOPFXC8jM6MBdF41lQM8MusV45HMiKJFvYkrkRUREpDX4vKCU/OIyhrXADiGdc3y0YRdDsro0KoneVbKX9LR2TXoGKxn01BoRERER2U9Wt451HrlPFjNjTHaPRtePd7O91J/u0hIRERERSUFK5EVEREREUpASeRERERGRFKREXkREREQkBSmRFxERERFJQUrkRURERERSkBJ5EREREZEUpEReRERERCQFKZEXEREREUlBSuRFRERERFKQEnkRERERkRSkRF5EREREJAUpkRcRERERSUFK5EVEREREUpASeRERERGRFKREXkREREQkBZlzLtnL0OTMbHtGRkavESNGJHtRRERERKQVW7FiBSUlJTucc70TPe/WmsivBboDuQme9RFh+O8EzzfVKW6No7g1juLWOIpb4yhujaO4NY7i1jgHGrfDgN3OuUFNszj11yoT+WQxs/cAnHPjk70sqURxaxzFrXEUt8ZR3BpHcWscxa1xFLfGSeW46Rp5EREREZEUpEReRERERCQFKZEXEREREUlBSuRFRERERFKQEnkRERERkRSkp9aIiIiIiKQgHZEXEREREUlBSuRFRERERFKQEnkRERERkRSkRF5EREREJAUpkRcRERERSUFK5EVEREREUpASeRERERGRFKREvgmY2QAze9jMNplZqZnlmtntZtYz2cvWVMyst5l908z+YmarzazEzHaZ2Rtm9g0zi9mWzGyimb1kZjvMrNjMPjazH5hZWi3zmmlmC8P7F5rZO2Z2aR3Ld6mZLQvld4X6Mw/0czcXM7vEzFx4fTNOmWaPg5mlhe/j4/Cd7gjf18QD/YxNxcxOMrM/m9nmsH5tNrP5ZjYjRlm1N8DMvhBitCF8rzlmNs/MTohTvk3EzczON7O7zGyJme0O698TddRpkbFJ5LrbkLiZ2TAzu9bMXjez9WZWZmZbzewFM5tax3yaPQZmlmFmN5nZSjPbY2Z5ZvaMmY2of0TqpzHtLar+Q7ZvOzE0TpmExMDMepnPa3LN/w5vMp/3DKjv56mvRq6nFtrPwhCDEjNbGz7X4XHqtI725pzT6wBewBBgK+CA54FbgdfD//8Geid7GZvoc14RPtMm4EngFuBhID+Mf5bQwVhEnXOAcqAQeAj4dYiJA+bFmc93w/RtwD3A/wLrw7i5cerMDdPXh/L3ANvDuO8mO3Yxljc7xK0gLOM3kxEHwIB5EW311+F7Kgzf2zktIFY/Dcv3OfAIcDNwP/AucJvaW8zl+1XEZ3ow/CY9C5QBlcCstho34MMwvwJgRfj7iVrKt8jYJHrdbUjcgD+G6Z8C9+G3Fc+F5XLA7GTFAOgIvBHqvBvWlaeAvUARcFwy21tU3bMi6jpgaLJiAPQGVoY6r+F/U54P/28FBid5Pe0EvBgRh7tDu3sMyAFmtub21mSBb6sv4JXwJX0vavxvw/h7k72MTfQ5p4UflnZR4w8G1oXP+qWI8d2BPKAUOCZifCdgaSh/YdR7HQbsCSvTYRHjewKrQ50ToupMDONXAz2j3mt7eL/DDuSzN3EcDXgVWBN+CPZL5BMVB+CiUOdNoFPE+Anhe8sDuiUxVheE5ftHrOUAOqi97ReTg4EKYAvQN2ra1LDsOW01biEGw8J6OIXaE9IWGxsSvO42MG6XAWNjjD8ZvzNZCvRLRgyA60KdeURsy/A7bFU7H+3qikdzxC2qXhZ+Hf4jsJD4iXxCYoDfIXPAb6PGzw7jX07WehrK3xPK3Bzr+yNiW9Ea21uTBb4tvoDB4ctYG6Phd8PvqRUBXZK9rM0ch+tDHO6KGPf1MO6xGOWnhWmLosb/PIy/KUadmO8H/CGMvzxGnbjvl8RYfR9/VHQyMIfYiXxC4gAsDuOnxqgT9/0SFKd2+CMpRUBWPcqrvfllOC4swwtxpu8GChQ3B3UnpC02Nslcd+uKWx115xN10CdRMcAnhZ+F8YNi1In7fomOG/AXfCLfm9oT+WaPAdAFKMbnM9GJajt8/uNo4qPy9Y0b/qqICmAZUVcF1PKeraq96Rr5AzMtDOc75yojJzjnCvB7bp2B4xO9YAm2NwzLI8ZVxeblGOUX438YJppZx3rW+XtUmQOpkxThmrhbgTucc4trKdrscQhxn4j/HpY0YD6JMhEYBLwE7DR/zfe1ZvZ9i32dt9qb9x/8Uc9jzaxP5AQzm4w/wPBqxGjFLb4WGZsUWHdrE2tbAYmJwRBgILDKObe2nnUSzswuA84FrnDOba+lXKJicAKQAbwZ8ppqIe+ZH/6t9f6HZnQRfofiMaC7mc0ys+vM7Nvx7iuglbU3JfIHZngYrooz/T9hGPNGi9bAzNoDXwv/Rq4UcWPjnCvH78W3x5/VqE+dzfijswPMrHOYdxegP1AYpkdrMfEPcXocfxnS9XUUT0QchgJp+Mssojeq8eok0oQw3Aq8D/wffifodmCpmS0ys6yI8mpvgHNuB3AtcBDwLzO738xuMbNn8BvcfwDfiaiiuMXXUmPT0tfdmMzsUOAUfDK0OGJ8omLQ4rfXIUZ34I8+P19H8UTFoKXHrWpbkYm/ZPVx/CU29wGrzOwei7gxvTW2NyXyByYzDHfFmV41vkcCliVZbgVGAS85516JGN+Y2NS3TmbUMBXifyMwFrjMOVdSR9lExKGlx65vGF6BPxp0Kv5o8ij8fSmT8dcdVlF7C5xztwPn4ZPMbwE/wd9vsB541DmXF1FccYuvpcYm5eIZjmg+ib/5b45zbmfE5ETFoEXHzfyT3x7DX8Iyux5VFDevalvxc+CfwGj8tuIUfGJ/JXBDRPlWFzcl8s3LwtAldSmaiZnNBn6Ev4P7koZWD8OGxKax8Uxq/M3sWPxR+N84595qircMw+aMQ7LbbtURFAPOd8695pwrdM59CnwR2ACcHOcym1jaUnv7Mf4pNY/iT+92Acbj7zl40sxua8jbhWGrj1sjtNTYJHvdrSEcDX0cmAT8Cf+0kMZo7hgkO24/xN8Q/K2oHZ3GSlQMkh23qm3FZuCLzrnlYVvxOnA+/p60q80svYHvmzJxUyJ/YKKPrkTrHlWu1TCzq/CnAP+Fv1ljR1SRxsSmvnV217N8XXvEzS7ikppV1DwqUJtExKGlt92qDVmOc+6jyAnhjEbV2Z9jw1DtDTCzKfhHnP3VOXe1cy7HOVfsnHsfvwO0EfiRmVVdDqK4xddSY9PS191qIYl/An9G6Bn8o0+jE5dExaDFxs3MhgG/BB5xzr1Uz2qJikGLjVtQta14Ofpsd9h2rMUfoa96bnura29K5A/MyjCMd43TsDCMd41USjKzH+Cf07ocn8RviVEsbmxCcjsIf8NTTj3r9MMfWdzgnCsGcM4V4ROTrmF6tJYQ/674zzMC2BPRuYcDfhbKPBDG3R7+T0QcVuPv9B8cvo/61Emkqhjkx5le9eOdEVW+rbe3qs5MFkRPCJ9jGf53f2wYrbjF11Jj09LXXaA6Rk8DF+Kfnf3VWNcXJzAGLXl7fST+sqPLI7cRYTtxcijznzDu3PB/omLQkuMGDdxWtMb2pkT+wFRtLKdbVM+mZtYNfyqxBHg70QvWXMzsWnznCR/ik/i8OEVfD8MzYkybjH+az1LnXGk965wZVeZA6iRSKb7TiFivD0KZN8L/VZfdNHscQtyX4r+Hkxown0RZjE+ShsU5JToqDHPDUO3Nq3qCSlac6VXjy8JQcYuvRcYmBdZdwjr7LP5I/B+AS5xzFbVUSUQM1uAfNnC4mQ2qZ51EySX+dqLqQNm88H8uJDQGb+PzmEkhr6kW8p7p4d/9Dh4kyGthOCp6Qrg3oyphzo2Y1Lra24E+v7Ktv2gjHUKFz3RD+Ez/BHrVUbY7vjfOhnSmMogU7WimkfGcQ+znyCckDtSvg4vuSYzPE2H5fhE1/jT8dY/5QA+1txrL9+WwfFuA/lHTzgxxKyH0ON2W40b9OoRqkbFJ5rpbj7h1BP4WyjxIPTq8SVQMSHCHUA2JWy31FhL/OfIJiQH7OoT6TdT4ZukQqoHtLR2fNFcCp0VN+0Wou7A1t7dmCXxbeuFvJtsavpTn8d0Cvx7+X0nYYKb6C7g0fKZy/BH5OTFel0XVOZd93Zs/CNxGRPfmxOi8AfhemN6Q7s1/E6ZHdrW8LYxLSNfvjYzpHGIk8omKAzW7nF4Rvp9m6+a9EfHpi39El8MfoZ8blrcc/zzqC9Te9lu2dvhHTDr8ddiPEa6Zx2/oHPD9thq38FkfDa+Xw7zXRIybG6N8i4sNCV53GxI34JEw/XPgJmJvK6YkIwb4nYw3Q5138U9dewr/e1IEHJfM9hbnPRYSP5FPSAzwHVOtDHVew+c5z4f/twJDkryenoh/rGl5iMdcYFGolwcc3prbW5MFvi2/gGz8j9dm/Cnrz/A3gtZ61DqVXuxLOmt7LYxRbxKhUx/8kcBP8Hfnp9Uyr7PCSlgQGvu7wKV1LN+loVxRqLcImJnsuNUzpvsl8omKA/4RhT8M30tJ+J5eAiYmOz5h+Xrhz26tDevWduAF4Pg45dt8ewM6AD/AnxLfHTYyefhn8U9vy3Grx+9YbqrEJpHrbkPixr7Es7bXnGTFAH+t9E34gwSl+B2OecDIltDeYrxHVTz3S+QTGQP8b/Ed+PymDJ/vPAwMaAlxA0bin4qUF5ZvPf5MQtzlay3tzcKMREREREQkhehmVxERERGRFKREXkREREQkBSmRFxERERFJQUrkRURERERSkBJ5EREREZEUpEReRERERCQFKZEXEREREUlBSuRFRERERFKQEnkRERERkRSkRF5EREREJAUpkRcRERERSUFK5EVEREREUpASeRERERGRFKREXkREREQkBSmRFxERERFJQUrkRURERERSkBJ5EREREZEU9P9GSaowpfaIRAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 377
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses['train'], label='Training loss')\n",
    "plt.legend()\n",
    "_ = plt.ylim()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvIAAAH0CAYAAABfKsnMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd8FGX+B/DPJEACIXQVRBBQUawIigVFgbN72E9Fz353evoTKx6enp6eWJATC4qAgoqnoiJNpfcakkCoSSAkIYV00uvuzu+PFJLts/tM/7xfL15sdmdnvrs7u/OdZ77P80iyLIOIiIiIiMwlQu8AiIiIiIhIOSbyREREREQmxESeiIiIiMiEmMgTEREREZkQE3kiIiIiIhNiIk9EREREZEJM5ImIiIiITIiJPBERERGRCTGRJyIiIiIyISbyREREREQmxESeiIiIiMiEmMgTEREREZkQE3kiIiIiIhNiIk9EREREZEJM5ImIiIiITIiJPBERERGRCbXTOwAtSZKUDqALgAydQyEiIiIiaxsAoFyW5YFqbcBWiTyALh07duwxZMiQHnoHQkRERETWdeDAAdTU1Ki6Dbsl8hlDhgzpkZCQoHccRERERGRhw4cPR2JiYoaa22CNPBERERGRCTGRJyIiIiIyISbyREREREQmxESeiIiIiMiEmMgTEREREZkQE3kiIiIiIhNiIk9EREREZEJ2G0eeiIiILMDlcqGkpAQVFRWoq6uDLMt6h0QWJkkSoqKiEBsbix49eiAiwhht4UzkiYiIyFRcLheysrJQXV2tdyhkE7Iso7a2FrW1taiqqkK/fv0MkcwzkSciIiJTKSkpQXV1Ndq1a4fevXsjJibGEEkVWZfL5UJVVRXy8vJQXV2NkpIS9OrVS++wWCNPRERE5lJRUQEA6N27N2JjY5nEk+oiIiIQGxuL3r17Azi+D+qNez4RERGZSl1dHQAgJiZG50jIbpr3ueZ9UG9M5ImIiMhUmju2siWetCZJEgAYpnM1vwFEREREREFoTuSNgok8EREREZEJMZEnCoNRLq0RERGR/TCRJwqBLMuY8P1OXPb2GqxNLtA7HCIiItO66KKL0LlzZ73DMCUm8kQhWJtSgEW7cpFXXouH5+7QOxwiIrIZSZIU/Zs7d66q8VRWVkKSJNx8882qbofa4oRQRCFIK6jSOwQiIrKx1157zeO+adOmoaysDBMmTEC3bt3aPDZ06FCtQiMNMZEnIiIiMpnXX3/d4765c+eirKwMzzzzDAYMGKB5TKQ9ltYQERER2UhhYSFeeOEFnHnmmYiOjkb37t1x3XXXYd26dR7L1tTU4P3338fQoUPRrVs3xMTEYODAgbj99tuxYcMGAMAnn3yC2NhYAMCvv/7apqTn/fffDzlOp9OJjz76CMOGDUNMTAw6d+6MSy+9FF9++aXX5VevXo0bbrgBffv2RVRUFPr06YORI0fi3XffbbNcbm4uJkyYgMGDB6NTp07o3r07hgwZgkcffRRZWVkhx6sHtsgThUAGR6shIiLzSU1NxZgxY5CTk4PRo0fjpptuQnl5ORYvXoyxY8fim2++wfjx41uWv/vuu7FkyRJceOGFeOihhxAVFYWcnBxs2LABa9aswahRozBixAhMmjQJb7/9Ns4444w2z7/88stDitPlcuGOO+7AokWLMHDgQPztb3+D0+nEggUL8Oijj2Lbtm2YOXNmy/I///wz7rzzTvTs2RPjxo1D7969UVRUhP379+Pzzz/HSy+9BAAoLy/HJZdcgtzcXFx77bW49dZb0dDQgMzMTPz000/485//jH79+oX47mqPiTwRERGRTdx3333Iy8vDokWLMG7cuJb7i4uLMXLkSDz++OO48cYb0a1bNxw9ehRLlizBqFGjsG7dujaTIcmyjJKSEgDAiBEjcPbZZ+Ptt9/G4MGDvZb9KPXFF19g0aJFuPzyy7Fq1Sp07NgRAPDmm2/i8ssvx6xZs3DzzTe3vIbmpH7btm04/fTT26yrqKio5favv/6K7OxsvPLKK3jzzTfbLFdbWwuHwxF27FpiIk9ERESWMuAfv+odQtAy3rlJs21t3rwZ8fHxeOihh9ok8QDQs2dPvPrqq7j//vuxePFiPPDAAy2PRUVFecxoKkkSevbsqVqszeUzU6ZMaUniAaBLly546623cOutt2L27NltXockSYiOjvZYV69evTzua73OZt6ea3RM5IlCIMFYUzQTEREFsnXrVgCNNfLeWs1zcnIAAAcOHAAA9OnTB6NHj8bKlStx0UUX4bbbbsOVV16JESNGqJ707ty5E9HR0bjssss8HhszZkzLMs3uu+8+rFixAkOHDsXdd9+N0aNHY+TIkejTp0+b515zzTU44YQT8Oqrr2LLli244YYbMHLkSJx//vmIiDBf11Em8kQhYI08ERGZTXFxMYDG8pJff/V91aKysrLl9uLFizF58mT88MMPeOWVVwAAnTp1wj333IMpU6agR48ewuOsra1FXV0dBgwY4HElAABiY2MRExOD0tLSlvseeOABdO7cGdOmTcPnn3+OTz/9FABw6aWX4p133sFVV10FoLF1fvv27Xj99dexdOnSlvfhpJNOwtNPP42XXnoJkZGRwl+TWpjIExERkaVoWa5iJl27dgXQWH/+yCOPBPWczp07Y/LkyZg8eTIyMzOxfv16fPHFF/jyyy+Rm5uL33//XXic0dHRiIqKQn5+vtfHKysrUVVVhb59+7a5//bbb8ftt9+OiooKbNu2DYsXL8bnn3+OG2+8EXv27MGgQYMAAAMHDsRXX30Fl8uFvXv3YvXq1fjkk0/wz3/+E5GRkS0dY83AfNcQiIiIiEixSy+9FACwcePGkJ5/6qmn4oEHHsDq1avRt29frFixAjU1NQDQ0ortdDqFxDp06FDU1NRg+/btHo+tWbMGADBs2DCvz42NjcU111yDjz/+GM8++yyqq6uxcuVKj+UiIiJw/vnn49lnn8XSpUsBAAsXLhQSv1aYyBMRERHZwFVXXYVhw4Zh3rx5+O6777wus3PnThw7dgxA43jriYmJHstUVFSgqqoKHTp0aEngO3bsiI4dO+LIkSNCYm2+YjBx4kTU1dW12XZzic+jjz7acv/KlSvbLNesuVW/U6dOAIBdu3YhOzs74HJmwdIaIiIiIhuQJAk//vgjxo4di/Hjx2Pq1Km4+OKL0aVLF2RlZWHnzp1ITk7Gnj170L17dxw+fBhXXnklzjvvPAwdOhR9+/ZFaWkplixZgtLSUrz88svo0KFDy/rHjh2LpUuX4o477sB5552Hdu3a4Q9/+EPLlQAlHnvsMSxZsgRLly7Fueeei3HjxrWMI5+VlYVHHnkEt9xyS8vyTzzxBI4dO4arrroKAwYMQGRkJLZv346NGzdi8ODBuO222wAAS5cuxWuvvYYrrrgCZ555Jnr16oXMzEwsWrQIkZGReOGFF8J/ozXERJ6IiIjIJgYNGoSdO3fiww8/xC+//IKvv/4asiyjT58+OOecc/Diiy+2jMN+1lln4V//+hfWrVuHVatWobi4GD179sSQIUMwbdo03HnnnW3WPWPGDDzzzDNYt24dFi5cCJfLhejo6JAS+YiICPzyyy/45JNP8NVXX+Gzzz6DJEk455xz8K9//atNazwAvPbaa1iyZAkSExOxYsUKREZGon///nj99dfxf//3f+jcuTMAYNy4cSgsLMTGjRuxYMECVFZWok+fPvjjH/+I559/HhdddFGI76w+JFm2z+gbkiQlDBs2bFhCQoLeodiGLMt4d1kKMoqq8PKNQ9C/p7kuWfkyc0MaJv+W3PI3O1YREWmneXjEIUOG6BwJ2VGw+9/w4cORmJiYKMvycLViYYs8qWrp7qOYsT4NAHC0rAaLnrpC54iIiIiIrIGdXUlVy/bltdxOyi7TMRKxbHQhi4iIiAyKiTwFVFBeiwnf78SbS/fD4XQpezITXiIiIiJVsLSGApq0YA9WJxcAAAb0isGfLz1V54j052WiOSIiIiJNsUWeAmpO4gFgQaLn2KtEREREpD0m8kQhYI08ERER6Y2JPKlKZpE8ERERWYTRhm1nIk+KsDSciIj0JjV1VHK5FA7AQBSm5kReMkhnOSbyRCEwyPeXiMiWoqKiAABVVVU6R0J207zPNe+DemMiT0RERKYSGxsLAMjLy0NFRQVcLpfhSh7IOmRZhsvlQkVFBfLyGufHad4H9cbhJ4lCwOMFEZF+evTogaqqKlRXVyM7m6OpkbY6deqEHj166B0GACbypDImvEREJFpERAT69euHkpISVFRUoK6uji3ypCpJkhAVFYXY2Fj06NEDERHGKGphIk+KGKVzBxER2VtERAR69eqFXr166R0KkW6McTpBlsUGEiIiIiJ1MJEnIiIiIjIhJvKkCAtriIiIiIyBiTwRERERkQkxkSciIiIiMiEm8kREREREJsREnlQlg8PWEBEREamBiTwRERERkQkxkSciIiIiMiEm8kQhYMEQERER6Y2JPCkiKRxI3iwzu8qyjC82peOd35NRVt2gdzhEREREAbXTOwCiUKXkVSAh8xhuvqAPukS3D2tdy/bm4c2l+wEAZTX1ePv280WESERERKQatsiTKVXWOXDbp5vx8i978MaS/WGvb/am9Jbb38VlBVyeM9wSERGR3pjIkyKSwhRWrcqa3/YcRXW9EwDwU0K2SlvxzSQVQ0RERGRhTOSJvHjr1/1YtCtH7zCIiIiIfGKNPJEXszY2ltoMPikWQ/p00TkaIiIiIk9skSfyY3FSrt4hEBEREXnFRJ6UYS9PIiIiIkNgIk9EREREZEJM5ElVZpoQioiIiMhMmMgTEREREZkQE3kyrDqHE4t25SDxyDHPBwU3oEuSwvHx2YBPREREOuPwk6SIln1dv9yUgXeXJUOSgPUvjEb/np003DoRERGRsbFFnlQWetP1u8uSG9cgA+8tT277oM6j5yhswCciIiISjom8TaxPLcQdn23B3M3peocSEqWlL0RERERWx9Iam3jwyzgAQELmMdx4Xh+c2CVa54iU8RhVRucaddbIExERkd7YIm9DOaU1IT9XacM4E14iIiIidTCRJ3MSWGmzan8+EjK9jIxDREREZGBM5MnWymsb8NjX8XqHQURERKSYkERekqQ7JUn6WJKkjZIklUuSJEuSNE/Aev/ctC5ZkqTHRMRK1FpmUbXfx1kaREREREYlqrPrKwAuAFAJIBvAWeGuUJKkfgA+blpn53DXR/oQlQd7rIcJNhEREdmcqNKaZwEMBtAFwBPhrkxqHGtwDoBiADPCXR+RL3KAMwKOeklERERGJaRFXpbltc23BY33/TSAMQCubvqfqC2NEmyW1hAREZFRGa6zqyRJQwC8A+BDWZY36B0PtSXpPaWqYFZ7PURERGQfhkrkJUlqB+AbAEcAvKxzOGRkglrKWVpDREREZmW0mV3/BeBCAFfIshzyrEWSJCX4eCjsTrhm4XTJKK2uR8/OUXqHIobBS1x2ZZViaL9ueodBRERENmKYFnlJkkagsRV+qizLW/WOx8zqHS5c88F6jJi8Gj8lZAtdt/KZXVXKwAW1lAcqrQk2/A2phQKiISIiIgqeIVrkW5XUpAJ4Ndz1ybI83Md2EgAMC3f9RvfDjiM4XFgFAHjhxyTcOfwUnSMyrkClNcE+j51iiYiISGtGaZHvjMbhK4cAqG01CZQM4LWmZWY13TdNtyhNoqCiTu8Q1KdR4swaeSIiIjIqQ7TIA6gD8IWPx4ahsW5+E4AUACy7CZMZG49DbTkPRFRpDREREZHWNE/kJUlqD+A0AA2yLKcBQFPH1sd8LP86GhP5r2RZnq1VnHYWl16C9akFuOfi/ujXo1NY6zJ6HhzqCQKHrSQiIiK9CUnkJUm6FcCtTX/2bvr/MkmS5jbdLpJl+YWm230BHACQCWCAiO2TMv5S0IraBvzp88aLHmuSC/H7hCu1CUopjfJoltYQERGRUYlqkR8K4EG3+wY1/QMak/YXQIa3K6u05faBo+UejxsmsdW5qV+tUh+zkmUZSdll6Ng+Emf2jtU7HCIiIlsQ0tlVluXXZVmW/Pwb0GrZDPf7gly3bctqauqdePybBNw7cxuyj1Wrui2X4PzU7DXmwcZv98R+9YEC3Dp9M66btgF7ssv0DoeIiMgWjDJqDfnx8ZqDWLYvD1sPF+O5+Umqbstl0MzboGFRk8e+jm+5PeH7nTpGQkREZB9M5E1g+b68lttx6SWqbssVoEnebp08fZUS2e19UKKq3qF3CERERLbARN6G/KXqoktrVKNRHu3rSoDdS2mIiIhIf0zkTUDSsIepUUtrPJgkTDvi1QoiIiJtMJG3IX9pllnyeF9kWcYvO7Mxd3M6ahuceodjKrIs42B+Beoc4b1vVrpa0eB0YfraQ5iyPBmVdSwZIiIiYzHKzK5kEIFa5JVeHBCV0gU6wXA4XWgXGYG1KQV49ofGDsHVDU78/erTATTW/lfWO9Alur2giJTFZwYfrEzFR2sO4bQTYrDy2asQEcGW9R92ZGHK8hQAgMMpY9KNQ3SOiIiI6Di2yNuQ/xp5k2SkbjlmQuYxAMBbvx5oue+9ZY0JWL3Dhes/3ICL3lyFX3cf1SxEs/lozSEAQFphFdanFoa8HiuV1ny85mDL7c83HNYxEiIiIk9M5C0omFy8pt6JaatSMX3tIdQ7XC33m6azq+z3zzbmbctEan4l6p0uPPm/RFXDsorq+tDLa6xUWkNERGRkLK2xIQnAzA2HMW1VY2tjx/aReOSKgQACDz9pVBW1Dry/PAVphVUej+WW1oS8XsPMZEtERETkhi3yNiQD+GBVasvfU1ektNwWXVoja1Sq8/7yFHyy9pDXx8JJxs1SaWQkViqt4edPRERGxkTeBNROi1rnKno0yJfVNOC+2dtwy/TNyCqp9rpMoHKNlPwKn4+pMXyn1RM8XokgIiIyPibyNuQvRwvUIr/xYBHSCiuFxvPesmRsPlSMpKxSPD8/Sei6AW3mjrJaXs88vhFPaIiIyMiYyFNbbhnpin15Hovc+OFGHKuqb3NfSl4FJi3Yg9UH8hVvck1yQcvtuIwSxc8PSIVkjAmePVj9ygsREZkbE3kb+GHHkTZ/+81N3BLUv36T4LFIncOFOZvT29x3z8yt+C7uCB79Kh6l1fUez1Hqo9UHAy8UJCvVbBMRERE1YyKvA4fTBaeGxegv/bzH7+OhtDo63Z50rLqh5XZynu969WC2X1xZh/+uTPW+cAj8tZ6H2uLKlloiIiLSGxN5jWUUVWHUe2sx6r21yD7mvWOnO9FlHCJWp2Yrd2lNg8d9IhPnosq6oJe1awmNXV83ERGRmTCR19iE73cit6wWOaU1mPjTbl1iULsxuXXSbYSWa/ec9KVW73ughNUI8RMRERF5w0ReY0nZZS23EzKP6RiJd8E2xIpssXUfWlJ0Y7B7rKtbda4NlKgXVtRhS1qRaSfKIiIiIutiIq+jOofL52PfbMvE64v3oaCiVvXOmq0TaSXpakpeBfa0OjHxtj5RVuxXPhqOCD8nZmP8rO2YJrDzrTmEvs+xLIeIiEgb7fQOwO7Si6owsFdMm/u2pBXh1YV7ASDoOnolRORZSdll+HhN40yqcx6+WMAaAztaVoM+XTsqfl6En8wy2KTzo9UHMfK0nrhkUE/F2yciIiJSA1vkdTY/Psvjvh/js1turzpQ4HfW0lD4ay8PNsnfkFrYcvvhOTvCiscjBh/Z9dGy2tDW5+cxJTXwd8/c5ntoTRbTExERkcaYyOvMCPmfEWJQlcBaj61pxcLWZWQsjyEiIjI+JvI6U6OePBAtc7RgXp+eJxJMWImIiMisWCNvIDmlNaipd4S9noBDKoa9BXX5Cj/UhF9UaQ3ZD3cPIiIyMibyemvKFA7mV+C6aRvgkoEeMR10C8dXfbqZ+XpJtQ1O3DJ9s7bBmIT19gIiIiLrYWmNzppb/J7/MQnNQ5WXVPnoUKlyDAAgi2iiDnNCKIdGY7bP3HBYk+2QefGEhoiIjIyJvEGITN4DJc/+kpNZG/VPbr/akuH1/rk+7g/E1zj861uNvBMulmAcx+TX2spqGsSc8BMRUdiYyOtM1AGxwenC2uQCFFfWocHle6IpwH/SmZpfGXYsSl+R+/LfbMv0utySpNyQ4vFVWhPKe8/0JTArvUdWei0izI/PwvA3V+KW6Zvh5GzHRES6Y428Rby2eB/+t/2I3mEYkq8WYiv2BxCF7w15M/Gn3QCA3dllWLo7F7cM7atzRERE9sYWeYPIPlYT1vPDSuJVbFgzwhV4kTkp09vA+B7ZQ0F5nd4hEBHZHhN5jczacBiXTl7tcb8sA2XVDTpEpB9ZlhGfUYK49BJNam3t3LosyzLi0ktQVMmki4iIyGpYWqMBp0vGW78d8PqYDOCXndnaBqSyQLn5lrRi3Dd7OwDg60dGaBCRfU1fewjvr0hFbHQ7bJs0FjFRwX3l7XvqQ0REZB5skdeAy09mK8v6txhrPbvsX76Ob7n98NwdIZXf6DEjbuN2fdxvgBIib95fkQoAqKh14KutGbrGQkRERGIxkTeACI3zeLU3F+i8pM5xfFSdUEa+qHf4H5VHaTxKrD5QIG5lGqttUPa+kTX9GJ+FP83YilX78/UOhYiIwsTSGp3JkMVmmkHYeaRU1fWHM459IPN3ZOFfi/cqSkp9jSMfip8Ts/GPG87C2mQTJvQKLhvYuFuBpVXVOfBi08gzcRklyHjnJp0jIiKicLBFXgP+8qdQSjIenhOH8trQO8i+sXR/yM9VylsJTDhJ4sSfdytuWfa2PVmWQ07vx32yCfGZx0J8NpF+ymrEday30sleRW0DZm5Iw7K9eXqHQkSkCBN5A1BaWrM2pRBTlqWoE4wAgerXPVvItS8wv+g/q0JOxo+W1QqORiNWyrwoJCK/aUbtFxKKqStSMfm3ZDw+LwF7ssv0DoeIKGhM5DWgPLENbNk+E7ccaZxPejtRKq6q1zYII9CotEbvztsiWSlZJd/mbslouT1jQ5p+gRARKcREXgMBa8Z1znu0Tla0frkia+R90WsUHSK96P27RURETOQNIZTjYYnAFmU1U1BvJwnuCUBRpQ1bx/XAzEsxvmVERGRkTOR1NndLBuIyShQ/z+mSkVNao0JE4Qs8ao222RGTsSasE7E9LWZSJvOrczixNa0YdQ6n3qEQUQBM5DUQ6Ni5IDEnpPW+tmhvSM/Tm5bj5rPjWmi0PtkyKua9ZEePfRWPe2dtw6Nz4wMvTES6YiJvYqI7bNY2iGl9ad0C7i0P0rIz5B8/2aTJ9pyca8k0auqdIU1ERmQXGw8WAQA2HSqCi98VIkNjIq8BtTpCikpPnS4ZTpeMr7dmCFmfHVsxZ6xPw57sMpYuGFxCZglGTF6FUe+txTEbjlzE3ZOUYmkikbExkdeAWgdPkS3NY6auw+TfkoWtzx+tDwzfbs/UZDt//GQTZm08rMm2VGfRg/e9s7ajotaBnNIaTP7tQMDlmcSQ3bAxgshcmMhrwAw/i5nF1cLWFej1ap0bHS6s0mxbWp0MUWjqHcdroFILKgMuz5yGiKzK6ZJRUGHSCQ6pBRN5E0sIcWZSTTERIqNilk4UEL8m1uRwunDDhxtwyeTVmLdNm6vWpA4m8hpQ81Ll3hzzjcoSoeWwNTqwQkdKa39CROQLE3d7WJyUi9T8Ssgy8MpCc46AR42YyGtAzd/FCd/vxO7sUhW3IJ7Vk8T58Vl6h+AVj89ERAQAxZyI0TKYyJtcWmEVxn2yWe8w2rB7Z6nf9hzVOwTLcLpkTF97CG/9uh9l1Q16h0Nkefb+9SYyHybyGrB5XushwuJDgRj181byrms51r8/CxKzMWV5CmZtTMd7y8V2JE7KLsOfZmzFzwnZQtdrZEbdN8m4uMtYk1rDYpP2mMhrgd+XNgySI9qOGXfD2RvTW25/u/2I8PXHZZTg+R+ThE2GRkREpCUm8qQq72f9zOTJWKrqHHqHYDpGuWojmjVfVfDsXhppF5Lt93TrYCKvAV7Casuix/8W/LzJqETum1ZN+Kz5qojIqpjIa8CixzufjDYhlNas8HmH8xlZ/USNyAziM0rw7fZMxVebLPDzRUFgg5N1tNM7ALI2b0ktEz1rC+VEpriyDj1iOhiwXMMcB7ukrFLMj8/CLUP7YsTAHpps03ifFTXLKa3BnTO2Amic2frVm88OeV2NV16s81m7XLLl5zIhe2GLvAbMkQqI8/z8JNw/ezuOFFd7fdzqtXlWaJHX0gcrUzH8P6vw6FfxeodiWrdM34xvtx/Bnz7fCofT5XM57puBWeHX6YtWncS/2JTuZ0l7+XZ7Ji749wq8snCP3qEQCcNEXgNWrSX1paSqHpsOFeHp73d6fdzqDXlWuGQZzmek9Lkfrj4IAFiTXIBDBRVtHrPCe6m1ao7AExa773FWPlz985e9qKhzYN62I8gsrtI7HCIhmMhrwMK/i37tyvI+46zF83gKQ1Wd0ZJQ7q1EVlRSxZlNyRqYyJOqvA4+afUmeQqZ+/7iqwwrq6Qaf/06Hq8v3genS81TZfOdhlu5RZXU534VjLsTkbGxs6sGeGA9btneo8gprdE7DFVZ4fM2Sj8GX6U1z/ywCwmZxwAAZ/WOxT0j+msZFlmYMfZ8IqLgsEVeA1ZPXJV4fF6i3iGQBTQn8QDw+948HSMxHn8XvCxwjklERK0wkdfA1BUpeodAGrJCsmSU6iejXBkwEytcEdKT3d8+7j9E5sJEXgMu/jLaCz9uYThqDZG+ePgiMjbWyGugwWnfX0K7Db0JGDf5NMNHYcf9xayseq3Eqq+LrKfe4cJ3cUcAAOMv6Y/2kWybtSMm8hqISy/ROwTdJB7xPgQl6UuWZczemI6c0ho8NeZ09OocpXdIYatzOPHlpgzIkPHoFQMR1S5S75AMR+SJEk+5iPQ1Pz4Lry3eBwCIjJBw/6Wn6hwR6YGJPJFgRm1Ubl33vnJ/Pt767QAAoLCiDtPvG6ZTVP4pqZGft+0I3l2WDABoHxGBv4wapFZYZGEG/foSeXhl4d42t5nIN5JlGeW1DnTt2F7vUDTB6zBENtH6BON/TZdjAeDXPUc9Wmpo8LUcAAAgAElEQVSNUl4QTJlS8wnKm0v3t9zXfJJC6jHKPkJiuTdEGLVUkMiXv3ydgKFvrMCn6w7pHYommMgTWUh1vQM7MkrCniTJTIfu5sTDKCPtGJmZPle9WGE34nchMLt/F4x65ThcyXnlWHUgH7IMvLfMHiMGMpEnEkyv30dZlnHLJ5tx14yt+OcvewIs6//vcOiVRGixWZeqs8gKYoIQSV3hfJ/ZAk9mVl7j0DsEzTGRJxJMr5FXdmeX4WBBJQDg+x1Zfpd1j9DjbxMdy5tPHCQVziDc34dah1P4NohIe3a/aMGrNtbBRJ7IIhqcrqCXdT/ZMMqwj8aIwjeDvE3++ZvZ1QzxU9iYpFEg/C2wDibyRILx91EfauQupkyI/O6AxjyBI+NQs+yOiMQTkshLknSnJEkfS5K0UZKkckmSZEmS5ilcR09Jkh6TJOkXSZIOSZJUI0lSmSRJmyRJelSSJJ50kCmY8cDnWWqjz4tYkJgNh4IrC62pkXSb8bMkIiL7EJUcvwLgKQBDAeSEuI67AMwCcAmA7QCmAfgZwLkAZgOYL6lRBEtEhklY5207gsVJuXqHYW4KSmuM8rmTcXCXIDIXUYn8swAGA+gC4IkQ15EKYByAU2RZvk+W5UmyLD8C4CwAWQDuAHC7iGCJ7Kh1K7uRE7jn5ieF9Dwlk0dZmp/P1sAfOwVh5oY0PPBlHJKyOGM2ETUSksjLsrxWluWDchgFl7Isr5FleYksyy63+/MAzGj68+owwiTShBmSJffSGY+/zfAi3Lnl8WmFlfrEYRO8Pqqt5LxyTP4tGRtSC3Hbp5v1DoeIDMIsdecNTf/bb4BQMh+DZsGtW6xVHUfeIC3jT8xL0DsEw/GctZPMonUrvJrTGbADNJG5tNM7gEAkSWoH4IGmP5cF+RxfR/CzhARFZDBlNQ2od/jvJNq61b2goi7AsqHTq6Os++lDar74Fnmzpzic7IeIyFoMn8gDeAeNHV5/k2V5ud7BEAWidaq0Na0YD82JQ12ARL61QwXWKzthqYdyja2v1nvjXC4ZiUeO4czesYiNbq93OKbCUz0iczF0Ii9J0tMAngeQDODPwT5PluXhPtaXAGCYmOiIvNP6yvS9s7YFtZy/khezlNYk55Vjxro04eu1y4BYdqmaeOu3A/hiUzr6duuI9S9ejXaRZqkiNR677DNEZmXYXzdJkp4E8CGA/QBGy7JconNIRKbmr6zi/RUpbZfV6Oi9IbVQ0fLjZ23Hwl3eh6cM5wTCLnXBNnmZ+GJTOgAgp7QGa1OU7WNEdmCTnwJbMGSLvCRJzwD4AMBeAGNlWS7QOSSioGlVh1zvcGHWxsNC1tWc+GgprbASD3wZp+g5JVX1Hvc1p+82aVQPSMn+Z4eDeaC+I2ahVSdyu5zsEVmF4VrkJUl6CY1J/C40tsQziSfyYt62TExZnhJ4wRBocSyfvVHMyUNzrOGkOb5KazxmvDVglqMkJiN3dm1wulR5f438mpWwyusgY2C7h3VonshLktRekqSzJEk6zctjr6Kxc2sCGlvii7SOj8gs1EriAeD7uCOGTFrV4uu1Wu09MOrMrgmZx3Dp5NW4ftpGVNVxlGEiomAJKa2RJOlWALc2/dm76f/LJEma23S7SJblF5pu9wVwAEAmgAGt1vEggDcAOAFsBPC0l1ayDFmW57rfSWQkRkmOwrF8Xz5W7M/Hdef0Drywzo6X1qjfxmSGj1ar/U/ku33PzK1ocMoorqrHBytT8crNZwtbt9MlI6+sFr27Rgtbpx40m5/BY64BM+z1pBQ/VesQVSM/FMCDbvcNavoHNCbtL8C/gU3/RwJ4xscy6wHMDSE+Is1YIZEHgP+uSNU9kVfyXvpLc2RZ9ujQq3Y8VifyrWhwHl9bcl6FwDUDE77fBQD4541D8JdRgwIsbZ8RjIjIGoSU1siy/Losy5KffwNaLZvhfl+Q65BkWb5aRLxEatIq1zN7vqFF/A5nY0fH3/fmYfpa38NW+vrMzJC3Kzm58JzZ1QyvUIy3fjsQ1HJWKKcK56tlp32CrMfsx8VQGK6zK5HVzdmcjjFT12F+fJbeoahC5A+ponV5WfbCN1bi9z1H8fvevJC275HTmSDH0exEUqPtENlNelEVdmeX6h2GKVngPFwxJvJEgvlr0attcOLfS/bjcGEVJv602+Pxkqp61DY41QzPVMItramoc+CJbxOD3Jb1jwDura02eMkUJu4j2jqYX4HR76/DuE82Y9neo3qHQybARJ5IQ3UNvse0XptcgEsmr8IV765BabXneOnu2CIqzriPN+H6aRuRVVLtdzkjlh0oGSKTSVlgVqiRD+dj5j6irxdbNfA8Pi+4RgiyNybyRFrykyM8PHcHGpwyiirr8Z6KQ0sGKyW/Ai6Xekf19YJn3IyICD0Byy2rRUp+BZ79YVeb+92TYiY51meHKzNkXBW1DXqHYGoWOA9XjIk8kUpW7c/H+FnbsGhXjuLnHjhaju/ijuBgvtgRPABlicqCncpjD1ZOaU3AZZT8KIv4/Y7PPCZgLcZlxhS1rKaBybVC4XV2JTIvO/5UMJEnEqz5h+Sxr+OxJa0YE77f1VL3HmxiuvNIKSYt2IPbP9uCmvrQauZFJD8v/JgU9jrCofePshn6unpcNVCwrNHN3JCGC99YgYfn7tA7FNsy1x5DZD9M5IkE81ZHXdk0W6XSlrKKWgfWJBe0ua+gvBYOpytgLe/cLRlt45Lb/q+XtSkFmLRgj7D1Nb8PVqht1pre+0Igk39LhksG1qUUajaKB/cjIjITURNCEVETNWdg/D7uCCb9sgcDesagwem74ywA/HvJftXiCFV5bQMenhN862owOVVzK7MW6ZfZWrTdmTn6shptaofN/hmHyy6v3x6v0n7seB7OFnkiwby1yIdzbGz9w/SPBXsgy43jDNc5/CfyRpRWUKloeUXDT/r5AQ85OfGYQMn4/L1UvXO03dmluPnjjXjxxyTbJIxEpB07/qwwkSfSgAwZDqcL7oPA6JHMqL1FEQ0i38UdCX57QTTBLN0d2njMZjgmKIvRvZ4+9FcYSgnKnZ9txd6ccvyYkI0lIX4mtqVRS6MZ9nkRbNhwSxbFRJ5IMAmSx3jk6YVVGPXeWlw9ZW2b+63YeiDiJSmpoT9+MiT+0GzG4Se1Gute6UnopAW7Ud+qHCzRoCMEsUa+LV45ITOx49eXiTyRYDJkPPW/thN5PPm/ncgtq8Wx6rZ1vq6mg6S/8dpF/y6Fc2DemlaMN5fuxyGFJTJasOMPeCCvLNyD0e+vw/rUxjH73T96rXK0tMJKfBeXpc3GwsTE1R7s/ilzN7cOJvJEKkjKLmvzd1Flndflmn9LH5wTp3JEjRxOF1a7jYITrKo6B+6dtQ1fbErHfbO3+VzOXz6tRmunli2ohpzZ1UdIceklmLftCNKLqvDgl3GQZbHRK3nfiysDz1R8fL2hREOiqJngldU04Pu4IzhcaLyGALIGO56gcNQaIh05XTIcTic2HizSZHuzN6Xjnd+TQ3pu61b4/HLvJyZ6Yv7XpOlAdrCg7WRi61IL0TnK+D/5gQ7EVXUOLNqVi8EndcZFA3poExQJ8crCvViSlIueMR2wZdIYRLWL1C0WK/xeSFLoiStPmK2DLfJEOhr9/jocLQs8w6kooSbxZqDGgcnjGGni1p7/LN3vWVqjTyhh+WBlKl7+ZQ/unLE1qNmBlbJ7jbz7dRuR+8iSpFwAQHFVPbYcKha4ZnsKp/XZqi3Xdvz6MpEnEkzJD+TRslqMmbre7zJG+WHSIw5fJUlaMUPiqyTxMkP9t6/9rHl+htmb0lvum73xsBYhkRp0/l0z/jeBKDhM5InIsNw7B/uj5kRcVuCtpVmrxN5bcu4rYfcVkuj5Gcj8KuscWJdSgNoGp96hEOmGiTyRYEbNLUKN66opa/HI3B1w+BlZxwjUKa0x3/CT/pg8fA9zt2SY4iqDqZjk7ZRlGffO3IaH5uzA/323U/Hz7X7ab5QrvRQ+4/d8IjIZ8YlF4y9ucZhlJp+tSwvpeZnF1cgsrkbPmA6Kn+t0yYiMOH7EMNuxw6g5Ym2DE+W1DTgxNtrjMV8xS34eC4VaiUCg0hp3v+/Nw43n9VEnGJMS+dkY9TuQX16HPTmNo4Ot3J+vczT6CKezq1E/V1KOLfJEgon+fSyvbcDyfXkoqNC3Xny325CagWxILcRF/1mJAf/4FeM+2YQth9QdmUdU7vLvJfsgyzJeW7QXdQ5Xm8eMMPxkWU0Drnh3DS57ew2W7c0Lr8NbGHFonQj4eu+/3pqhaRx60eokWP89PDgutx1w2d6j+N/2IyyzIdthizyRaIKPhBN/2g0AOLVnJ7ErViglvyLwQjhei/3Al8fHxt+dXYbxs7dj0ZMjVYmt9XbDNWdzBmrqnfh+h+cERkZoxfpgZSqKmsZlf3xeApLfvD7o5+p1ImK2KzFGZIBdz9Aen9c4CV9lXQP+Ouo0naMh0g5b5IkEc28pEiWzuFqV9RrV+Fm+J51qTY0k0VsSr7U6h/eWxbyyWr/P85usC5zZVcl5k7fNKO2czM7MvmWVVCMl7/iJthFOOEurg58ETKTJv1l3iF2zmL72EK6eshYLd+boHYotMJEnEswAx1BL2JIW3DjTWr7fTo06/E5dkYJzX1uON5fuF7ZOo3VuU3p1QKurCfnl/k+U9OLr4ztUUIFRU9biumkbhNSKizgJeHdZMoa+sRITvlfeCZXMrbiyDlOWpyCjuBrP/LBL73BsgYk8kWBqtciTf2pMDuTu7s+3ajJKysdrDqHBKeOLTemKa379hafXnqnmOYTojyMuvUTsClX24k+7W96Dv3wdD0DwSVsI729zx/pFu3JRqHPfHtLWMZ2uxNgZE3kiwZjHa0sCNJsdN7esFgeOBtdX4JM1B/HYV/E4GGTfAl9Enhh6rErHfZWlNWKUKZhrIRiir3z4KhEjfRmh4z6Jwc6uRILZLZFPDTNRDdex6npklWiTyANATRAt5NsOF+P9FakAgP25ZdgyaWzY203ILMF/V6ZiM6e2JyKiJmyRJxKsqt6hdwia2JVViofmxOHaDza0ud9fu6kaddqJR0rx/vIU8Sv24ZM1BwOWu2w8WNhyOzdA59Rg3fHZVq9JvPuJo6/zSAmS5wRXKrbKHThajhX78uBwugIvTKSQ0fp8mA2vcFkHW+SJBCsVfKnbqG6dvlnvEFrEZWhX17w2pRBfbErHk6NP97mMEQ6S3mLQ6mpRTmkNbvpoI1wy8MpNQ3Bh/27abJjC5nliaLNLjEQmwxZ5IiKFpmh4BUDpSYGvzrjeWjCnrz0USkgBvbcsGc0D/Pzn1wOqbIPMTf9TXWPyNx+GS+CoWTxBsw4m8kQklp8jtFqJo9FoedlfyQHZfclZG9OxP7dcbEAAHGEmHBW1DViTHP5Qiu5EJkIivLpwL0a/vw7rUwsDLitqwrNAtHqHWm9n6e5c/O2beOzQ8MqaUfk6EU88cgyXvbMat0zfzNlrqQ0m8kSkmeX7xCdnRpGSV4E/zdiKSQt2G7LD89GyWvztm3iP+/fmloW0vnDTSl95qSwD42dtxyNzPWP1Znt6Cd7+7QAqagOXtI2YvBp/mrEV9Q796/bj0kvwzbZMpBdV4cFWsyD7osWwp2oIdAJSXtuAp/63E8v35eOuGVs9Hldr7gat3s3aBiee/m4nHpoTF9boWvfM3Ib88jokZZXi06bhPcMhovwvs7gKCxKzUVl3vF+YSXdTU2MiT0QkwENz4hCXUYLv4rKwKEm/GQ2bD6QOV9tktaymAbUN4hLYcI/Xvg74RZV12JOj7OTi8w2H8eGqgwGXK6qsQ1xGCb7akqFo/Wo4WKDvaE/BCjcx81nq1fR/rp/5H+bHZ+GCf6/AcyaeWOjTtYewOCkX61IKMfGn3SGvp/XJ5z6F3w9vwi2tqW1w4rZPt+C5+Ul4ffG+sOOh0DGRJyKh7FL7mlVS3ebvo61Gp9FyOExvedLsjYfxr0X6HVzD2QdCTRxnb0oPetkDeeLLidSmWWmNRk2qwWxl4k+7UVnnwIKdOdgrIHltTavfqaW7j7bc3niwSOi69fytXZdSgJKqxsmffkrIbrlf9G4qyzKen5+EGz7ciJ1HjolduUVw1BoiEkqrhENvh4uq0K9HJ73D8JB1rFqTDqZKp3PyuMfHCrTYfYwwqpBdiPo9KKioBdBVyLqMzCy/n77O90SfBy7dfRQ/JzaeKNz26Rb87apBOCk2Gg+PHGCa90ptbJEnIgqBkpbLj1cHLvsQZfys7ZptSw2ssQ1fOAlOuO//sr1HAy+EwCeC2w+3nTNB9H6hR6deI6/TqHYeKW3z9+frD+ONpfuxYr91+1spxUSeiCgMJVX12BTgkvnUlak4XFgJWZZRU2/9ESfcE8mZG4LvnBduS/38+KyQt6ElNU9YRJbHKFnTrqxSPD4vUdF6fYV698xtCrZMWvM58VyI3y2lnZrnbA6+lM7qmMgTkVAGyJE0U+9w4bppG3D/F4FbwQ8VVOLW6Zsx7M2VWLY3z+dy3pKwZD813eGmbFp8XiJGKwomN12XUhBWh0JTMtAX7lMNh5cNtzxKq7dN6XbMOjpRs1DCf33xPpz3+nJ8oaCfiy8G+jpohok8EQklSc31rNYmA1ixPw+FFXVBLf/DjiwkZZehpsGJx+clBL2db7dn4vppG0OM0l4emrMjqOWMcLBX86qAVrXDDU4XFu7MwbK9eYoT0OYIjXB1RE3mTst9E/WxFVTUYu6WDFTXO/Hm0v1hr8+q77c/7OxKREKl5lfi8rfX6B2GJhqcwQ/neMRtlBtf3PMhw86MGmYGpvTZRkz4nC4ZPydmw+GUcddFp6B9ZPBtY2o1vNY5tCvd+iUxBxN/brwC8tUjIxQ9N1BpjdqMmvCZpQOnqNKailpH4IXIL7bIE5Fw4c7saRbhXt7fm1OG/65IweHCypDXodul+DC3O3tTOtKLqgQFo4yoXGlJUi4m/rQbL/+yBz/GZwd+ggaWJAXX2dQXJR9rcxIPABN/SvK6jDnSUvXY7fWbvDLIlJjIExHpYOqKFNz88SZ8tOYQ7p99vMY+TWFSH+609ot25Yb1/HA8/k3wJUZG9FKrRPblX/boGMlxSq4SBSPYE0VZ9n6C5LPlNvSQhNBq+8xrtaX3fqUHJvJERDr4eM3xjoG5rSaTmqNw1tFH5saHFcemQ0XYkx3CZDt+mrWDPZim5JtjdlNf1KyCKKmqx7M/7MIrC/egtiH4chk7JjKkrup6BxYkZiNVh++r0u+YHU+cWCNPRBQiJQeZQAeYg/kVOJBXoShpE+XnxGycd4r1J9tpZobL/28s2YeFTVdLToqNRp9uHb0vKHx8dbErNOqJhQl2AVUp+Q5MWZ6COZsz0LF9JLb/cyy6RLcXFodR9w8zYYs8EVEoBGcCN3+8CU9/txMLEnPErphUo+YMsQtblTx9v8P72Pgv/7IHh3XqZ+BO7cTYDCdfVjVncwYAoKbBiR/iAs/TQNpiIk9EZAB1DrG1zXoy+sAbZswJ3d/S5Lxy/G/7EeHbcU+YlbxXap7YiGaeSNUR6nc03Cs2Sid+UsqOnysTeSKiECkZKs7sE71oJTnP2HXzS5JyMXbqOkz3MvmRksQ63L3haJn152og9Wj9c1TvcOGOz7ZgxFursDWtWLXt2PFXlok8EVEIZMiWaf2x20lGOC/3/77bibTCKkxZnoIat/4MRhm5xg7Cvepj5j2++ftqljHnAWDulnQkZB5DcVU97p21Te9wLIWJPBGRBsycOChlhvTC4XThm22ZmLPZc1r4nNIa1bev5D1ScqJl5wS32cNz4rAg0f+4/pUKJiKqrnfgu7gjYQ/1agXNu6LSk+Hko8a+0mZmTOTJkE6MjdI7BKKATNQgFpQjxdWYsjwZ8QZNWCpqG4Sta8HOHLy6cC/+vcRzWngtxrf3lQdtSC3E8/O9T67U2lPfJooNyIdgEzYjXdRZm1KI5+Ynoaza9/4ybVVq0Ov7aPUhTFqwB3fN2Irn5yehpKo+6Oca9WqX1r9dvt6FqSv9fw6KZ4BWuLwVMJEnQ3r8qtP0DoHINpoPso98tQPT16bhzhlbUVVnvKnTP1x1UMh6fk7MxsSfdvt8fE9OCOPqC1Db4MQDX8bh5wCtyQBQVa/OMKWhJp5FlXVYti9PcDTHhRLV0XLfV1YSj5QGvZ4Z69Nabv+cmI03luwLIRpjCff8QsSJQHJeOX7dHd5MxO6MedqkLibyZEhWa+kkazLTKB3BOFRwfFbZA0fL/S6r9St/Y8l+7Mg8pvFWG8myjIU7c7yW4YTqkJfJdYoVtPQqIXq2V5EM2mDt10IFsyGbqY49GDKAvLJa/F3AFaHDhcYYOtXsmMiTIVnrp4+syIwJiBKhvDw1ywhS8iuQlBV8K6pI61ML8cwPu7yW4bgrra7HX76Ox2Nf7fBbgvHV1kyP+0S8f95OLp/UqAzHG1F57Pz4LGxJK1K27TCOJOW1DdhyqAiOME+CjFpaE45JC3xfzfLF2/sQocKB3o65AxN5MiSrtWKQNSnaTQ18PA8316iud+COz7bgqinrsD/Xf0u+Gb3ze3LQy779WzJW7s/HqgMFeHNp4MRfCyv257f5+7N1abh+2gYs91IKI3o39bdvuVxy0P0eVu7Px/hZ21uuGrmC2GlDPYy4XDL++PEmjJ+9Ha8u0reMRpZlfLRaTEmZKGtTCgWticd5EZjIExGFIC69BDnH1B/dRC/uedIuP63hH646iITMYzhSUo2H58ZZ7kRcyetpXd/+y87gZ+ldkJiNWRsOK4rLF39lUXlltXh3WTKS8yrwtyA69YY7AZAvDU4Z103bgLtnKhuK8INVqZBlGZ+uTQu8cIjiMkqQWVwNAPguLrxJt8L9LizZfRT/DdAh1Ay87UVq/EwYuL1ENe30DoDIG4vlAWRBnwtKusziTzO2er2/tLq+zXuRX16naL1TV6SEFZfRhJpIPOdnpBoZwf8mvrc8GUWVvkt6jpZpe/LpK+5vt2fiYKs+GcH6dfdR4R0k3YnsUxBMac2hgkqc0r0jottHejw2f0eWx33iWsT1xcO8GGyRJ0PiF5xIO95aXQsq2s4cWu8jufm/73aGte2P13jOkGo0Rvg9Crb8yV8SL3I7wXK6vK/waKn6M9OG+rlpWdY+Y30a/vDf9Rg7db3wTslrUwowd3O60BGoRL43aly5M8J3VWtM5MmY2CRPpCn3lsMJ3+8K6nkbD3p2QOS317i0Lnu6eso6r1cBtAyj3uEybKfT5v4XOaU1ikqxAknOK8fDc3bg9SX78dEa/Wvsvb394XR2dbmA9CKOegMwkSeDUuM3/u9Xc2x60o8x04jjNrgl5L5aUu3ICO0K2sUg+/1TKYdLxssL9nhuRaPda31qIYb/ZyVumb45qOXrHWJbxTOaau2DUeFlttlNh3yP1ONvl2h9pevz9eLKAN9dFnzH70CC2ad9LROXUYLR76/ziMeOv1qskSci0kBBufqlBKGSZeDLTd7HSP8xPgvz4z3rdP2uT0RQJuWv5besugHP/LATjhBOkvzV0BvdTi8dpbU4MZEk4MEv4wAAu7MDT/KVkFmCx76KxzE/M8IaiZm+Z947uwbeCQKd8H22Lg0vXX9WaEFZBBN5sg0jtKqRfak1E6cImcXVXlv+auqdeNHHDKj+vk+/7VG3M6JZvf37gZA6Kh4tM+5JoLH5/9FPK6zEaSd0bvn73pnbffYFofB4O8FV45Bsx8M8S2vIkNRoabDaLJxEovi6fF9UqWwEmmZ1gssT9KakEcBfK+PipOBnBNWLQUvJVeE+/KaSJL64sg5TV6Rg6e7QPtOtacX4aktGSM9tZvYjWjAt8myAC4wt8gZ0SveOyLbw+NRBUeFowh8EImWeN3E5h0hKGgHcWx7LaxvQJbq96JA0Y+a8PiNAZ8hDIQx/2eyVhXvx+97GCbUG9eqMs0/uEvRzs0qqce8sZePnK2WGw52/zq6ZxVXYkFqI4iBHYco+Vo1Fu3LRtWPb75osy5ab18IdE3kDsnsir9Z3ztpfZSLx4jJKfD5m9StcZdUN6NqpMSkI5zfpuR+SMPvBiwCYo7VbnauhnpLzKnwun1ksZjSSx76OF7Ieb5qTeACYH5+F18edE9TzFu3KwQyBnU/NwmuNvI/fEKdLxj0ztykqKXvsq3iv+9TipFzcMrRv0OsxI5bWGJAZfuzVJEGlViCLn5UTacnpcmH1gXy9w1DNFe+tQXlt+J0eVx3IR3pRFVbtzzfESEBm+BV8OsihT81owve7/M68a4bPRxRvh+SR76zBX7+OV9wvxNeJ4aJdxi9nCxdb5A1I/596/bkMcMAjIt/+8+sBy9XCt1ZR68CMdWmYKGBEjOumbRA+rGEoCipqMX2t8SfgSvIyyo1dBHvk25Ba6DH60dGyGvTp2lF8UCJ4eWHeEvmc0hrklIqrSFiTXIDaBqfXWXOtgi3yRsQclogMzspJfLPmFnklraTefr6NkMQDwOPfJGDFfv9XUfS+Ilxj4NGdjOSBpmE1W3vuB/36tNQ2+P7cXC7Z69UorcrzvvAxtK5VMJE3IJfev6Q6G9ArhucyRKQ7K/0U1ztcSDyivKVb6/fg0rdXa7tBC9l6uBhAYwdPf4m1Gsa8vw51DqfHsTv7WDVGT12HZfvyPJ6jVbXrvG2Z2mxIJ0zkDcjOiXx0+whMHz/MUgdQIiK9jQ9ylBRZ52aUshpzTMbUmsiEtCHMcexlWcb4Wdux6kCBoIiCk1tWix92eE4cN/Gn3cj0MbtthKA3btIC73Nd2AUTeQNS42c0ur3xP+pTe3bC9kl/wJA+wQ/jRUSklvSiKhwprkZSELOCGl185jG9Q6AAyqrrfSa9wVp9oKClZV5r5TUNHsUy2/zEIuoE6Ls4ZTNPW43xszsbUqM1+sTYaPErFaxrx/Ytw70peepDqAsAACAASURBVAu+fewSdQLyoWdMB023R0T62JJWjKveX6t3GIYhyzISMo8hPcD47BSaj9aE3xE5v0LfWYCVHLvTwhjHXwmrX+HnqDUGZMR9TpLU/zK0LinyNp2zLzFR3I2JSB1WTwLcub/e1qU2i5NyMeH7XZAkYPVzV2kcmbnoNRGRmeZ3+MeCPXqHYAnMgAzqnJO7YF+u77FmlTj/lK4orQ697nDhkyPRM6YD7pqxFXnl+p7texP0z5bdjshEpDmz/8wcLfM99N+EpvHdZRkYM3V90Ou0+sya7rYdLsaE73fi9BM76x2KpiTJTKcR1iGktEaSpDslSfpYkqSNkiSVS5IkS5I0L8R1nSJJ0peSJOVKklQnSVKGJEnTJEnqLiJWM5BlGR/ec6GQdfXuEo0P7h4a1jq6d2qPfj06qd4JyiwHQJOESUSk2F+/TtA7BNO7Z+Y25JfXYfMhfWrV9cTjo/ZE1ci/AuApAEMB5IS6EkmSTgOQAOBhAHEAPgBwGMAEAFslSeoZfqjG55JlYZ1ANr00GqedoH6rQEcBky20TuSVJPUndQmy/t9mrUJEREq5TzJEgc3ZnIFcgZMYaUFJ+Wo4jLA76T0Sk9pEJfLPAhgMoAuAJ8JYz6cATgTwtCzLt8qy/A9ZlsegMaE/E8BbYUdqAi6Bc4e0i2z8iMPJYZsvlvm7aBYZoV+SHN0+Al8/MgJ3DT8Fo888Qbc4iIisxixXSvV2+6db9A4BgP7tVcFuPoGjKAkjJJGXZXmtLMsH5TBO8SRJGgTgWgAZAKa7PfwagCoAf5YkKSbkQA3irN6xfh8fdIL5XuJVg8NPoOU2t5XtSqMGn4Apd10QdhlRMLRqySAi0tvCXTm4Z+ZWrPAyoU+wSqrqMT/e2kME5pXXYo+Jhin1129hQ2ohrp+2Ae8tS1a83mCPjnd8ZowTHysw0vCTY5r+XyHLcps2aVmWKwBsBtAJwKVaBybasmdG+T1rfu2P5xiyw4ivmJ8eczraR4qNONTLcd06dcC4C04WGgsRkV29tywF2w6X4K/fhFc7P/EnY07ac7SsBm8u3Y9Fu0KuCm7xx082Ydne0E94RAj2SOyvQeqBL+OQnFeBT9elYb+gQTdIPUZK5M9s+j/Vx+MHm/4fHGhFkiQlePsH4CwRgYrw2X3DfD52QmxUUOsQUZfu7sbzenvc15zA+/ren9AlWsioBKJaugf0Mt8VDSIi0t6zP+zCF5vSW0bkCdfj8/TtLNwguCg9rVDZWO9GbIS0OiMl8l2b/vd1bar5/m4axKK6a8/2TJiV2jBxdNDLBvvlmnqXZ2lKwPxaloV/ecPJ6dX+IWFhDRGRNWw7XKJ3CELN3yG2hEnp8Y7HR+0ZKZEPpDk/C7ifyLI83Ns/AMoLvlQSIaBzaDsVOph27BCJR0YObHOfsymr1rITjZIaeRHvJRGRHW05VISqOgfWpxaizuHUOxwK056c4Or07Ta2v5UZaUKo5r2vq4/Hu7gtRyp59poz8OXm9Ja/ncFcqhPwm9C6Fd7pDD6R7xLdvm0o/H0iIgrK+Nnbcf4pXbE7uwzXnH2S3uGQRoItZf3vihSVI1Gf1cenMFKLfPPe4qsG/oym/33V0NuOWglrrFtiHMwXPkJwMM4gv3k3nddH6HaDYfUfBSKyl91No62s3J+vcyRkNBnF1XqHQAEYqUV+bdP/10qSFNF65BpJkmIBjARQA2CbHsFpzUiXvQIl1TIAEdUtrctpgroKACgaLaeD4JF1iIiIzCitsKrN3z8nZGNPThn+OmqQThFRqDRvkZckqb0kSWc1zeLaQpblNAArAAwA8KTb0/4NIAbA17IsV8Fk1BhdJhiv3DREyHocQZS5+JssKlitc/dgE3klsYw+68SQ10lERGQFtQ1O/LKz7XCbz/+YhLlbMvDMD2JG7yHtCGmRlyTpVgC3Nv3ZPBzLZZIkzW26XSTL8gtNt/sCOAAgE41Je2t/B7AFwEeSJI1tWu4SAKPRWFLzTxHxWoW/5LlnTAc8dqWYM2tXc2dXf7EIbuwOJ5GPau95fvrk6NPQLsL3eevTY07HR2sOhbxNIiIiM1iXUujzsbj00EfxKayow5Kk3JCfT6ERVVozFMCDbvcNavoHNCbtLyAAWZbTJEm6CMAbAK4HcCOAowA+AvBvWZatNU6UitwT63BKdYJJqkWPIx9OIh8T5blbd4lu73cknOeuPZOJPBER2YA6Hb3mbslQZb3hsnq3NiGlNbIsvy7LsuTn34BWy2a43+e2rixZlh+WZbmPLMsdZFk+VZblCUzivfCbO4trIncF0bvTVx6f/vaNQW+n9VYcYSTyvWI6hPS8k7p4n4hr4vVntvlb1MRVREREWuMhzFqMNGqNZUV7KfUwuvNPaRwFVJKAs/v4GhH0OF+nDaG21IfTIn/N2Sehf49Obe7zt7bxl/RvXMbLQv17dMLN550ccixEREREajFfhmlCM+4f3nJ7zsMXB/WccNvTw610mXH/cLxw7WAsenIkOnYI3FlXSI18q0T6Txf1C3k17SIjsOyZK4NadlCvGPzjhrO8PrbquVFY8ewoRHK0GyIiMrmCilrsyCixfKmJ3Rhp+EnLumRQTyx6ciQanC4MP7W7sPX6S57DTT1P7tYRT405o819vlrXZVn8OPLnndIVH9x9Ab6Py8L2EDrfdOrQdteW4NniHvfPsTgxNtrnOk4/MVbxdomIiIymrKYBo6esQ1W9ExecEvgqO5kHE3mNXNCvm94hqEpEIu/eSnDbhaegf49OuOOzrULW7Z7I+0viiYiIrGLWhsOoqncCAJKaJgAja2BpjYkpSZ3NUBzirRNpoE45Ii8RBrsuXpYkIiIzqaxz6B0CqYSJvI6i2qn39rs3kHfr1F61bXnbnijdOoU2Ak0oLh5wvOzpzJNYVqOFW4ayIzERkdocLpfeIejG6qP0MJHX0Q9/uwxn9VYnYXSfLOq9Oy9QZTtAY0u6GqU1AHD6iZ1x69CT0SEy/F31RB/DSzb797hzcfqJnXFK946Yft+wsLdHgZnhShERkdk57ZvHWx4TeR0N7dcNy54ZhcsG9fR4rEtH/y3oPWI6+B3a0f2h00/sHFKMwRIyaI2Ps+Zp91yI3a9fi+vP6e19gSD16hyFV24agnNO7tJmJKFmJ8RGYeWzo7D+xdFt3i8mm0REZGaHCir0DoFUwkTeoHrEdMDfrhrU5r4P7xmKs3rHoneXaHz9yAi/z/eWfLYPMIziR/deqDTM49sTkO36m3k1un0k7hx+StjbeOzKQfj16Stx/bneTwokSUJkRNsX4xGVxS/TERGRtezIOKZ3CKQSjlpjAL6S4Ek3DMEjIwfio9UHcWrPThh3wckYd8HJcMlAZISEKoWdVxrLbXxnoTed10fR+tqsW0RpTYAEOcIgp53n9u2KrYeLAQCDTojBgJ4xWJNcoHNU5iRivyEiIvLN2q1vBkmN7C0myvf51EldovHWbefhr6NOgyRJXluMg+WvxRtAGOtVr7OrEbi/tPfuPB8nxkahR0wHzLh/OL586GKMPetEXWIjIiIi34oq6/UOQVVM5A3gXzefjeYc+v27xHRKVaOl098qhXR2DTTUpJfH9eiN3q9HJ2z+xxhsnTQGg5tGt2nH2V+JiIgM6bN1aXqHoBqW1hhAvx6dsP7F0SisrMOFCiaOUpo7ByqtCcRf0qxXGhvqVQQlvL3s9gJG0SEiIiL1vbssGU9cfZreYaiCibxB9OvRCf16dNI7jJDpVVqj5lj8pD5exyAiIgodsyATcx8rvs1jQWRI791xvrLt+VinLAOx0eFPOOVtZtdAtGgZDybZfPCyAWqHQURERNQGE3kT89d5NZhEvn9PcVcAHrjsVNVnj/Wmg0Fa5C8a0EPvEIiIiMhmjJEFkS6UJsH+Tg46dWiH9S+Oxr0j+occTyjV+1ok8j07d2i57es9MMoJhemwtoaIiChkzD4syl/ZTbML+3XDmU2jrtx/aeAEPFDlS9eO7XFu3y4e95/SvWPAdQezfm+uPKOXautuFtUuEt/95VLcd0l/LHnqitBXRERERLqobXDqHYIq2NnVxJQmp1HtIlDvdLX8LUkSFj01Esl5FTi/b1fVYprz0MV4Y+l+bDxYJGQbzZ79w2BcflpwiXy4LjutJy47racm27KTYE44iYiIwvXl5nT8/erT9Q5DOLbIW5S3EpDP/zy85fbchy8GAES3j8TQft0QEcQwjsHU3Xs7tzjjpFh88+glQTzX/5mJ+0nChD+cETigJlaesMrMnC5X4IWIiIjC9Nlaa44lzxZ5i/KWt152Wk/8/MRlcDhljBhovM6Zak7upMfEURTYb3vz9A6BiIhswKppABN5G5EkCcNPDT2B91UGIerLYdUvGflW72CLPBERUahYWmNiWie+gUpfGhdSZ+ZYIiIiImqLibxFSToVhTMXJyIiItIGE3kT69Q+ErHRjdVRvbtEt3lMjTRe/RFGeBpA4Xtk5EC9QyAiIoMJZfZ4M2Aib2IRERIWPHE5nrtmMP73F7dRYTRskG/95QjnexLouSd2iQp95WQbHKGIiIjsgp1dTe6Mk2JxRtOkTq3plcuoecZ7/indMO6Ck7HqQD5e++PZqm1HK0P7dcOurFK9w7Ac5vFERGQXTORJKH9p/AmxUSisqAvpuc0+uvdCNDhdaB9p/otJF/ZnIq8GtsgTEZFdmD8bItOY/cBFaB8poUO78HY7KyTxADCgZ4zeIRAREZGJWSMjIg9qjFoT7iov6NcNWyeNxfZJY70+btWOKN6MGNjDVq9XS3qN2ERERKQ1JvIWpV+NvP/He3WOQveYDt6fKziWR69oHL2kY/tIjL+kv+C1h0nmGD1ERERaseoxlzXyFqVXo6SRvigvXHsmzu3bBeec3BVdO7bXOxzyQpI4ERgREVGomMiTYYhO6Dp2iMRtF54idqWCyJCZwKLxypHot4ElS0REZBcsrbEo9SdvOq513hROEhUVZidYM5FZWgNAnXp25vFEROTOqscG+2ROFDa1Tw0+vW+YylswFrYcc8x3IiLSRk2DU+8QVMFE3qLUqJH3NvGUu1By0z5do7HoyZG4aECPEKIyJzOm8G/eco7wdUao0SIvfI1ERETGxESegvafW89Fr85RiI0S27VixMAeuKBfN6HrNDoztsaPG9pX6Pruv7S/Kk3yJnxriYiIQsJEnoJ2UpdobPnHGGz/p/dx4IHGTpxK2bW8wmwJp+jG816do1T57EPZB4mIiMyIibxFqTUpTod2EejUoR2uOfuklvtuvqBPWOu04wQ+dk81Y6Pb4bErB+GE2Cjh6zbbCRIREVGoOPykRamdGr99+3m44JSuuLB/d/Tp2rHl/lCSKBvm8U2j1hgr49z+8lg8+W0i4jOPqbqd+y7pjxevOxOdo9qpUyPPTJ6IiGyCLfIWMvH6M1tuv3zjEFW31atzFJ4acwZGnt6rzf1XnHH8777dOro/zSsth8o0EqPlmyd1icZPT1yu+nbO69sV3To1zu6rxsmMwd5WIiIi1bBF3kIeGTkQsVHt0D2mA0ae3lOXGM45uSveuOUc7Mg4hgljTw/qObZskQdw9Zkn4u3fk/UOJWiiPqbWibYaJzMuo50hERERqYSJvIVEt4/Eny8boHcYeOCyAXhAQRw2zOMBWcaZvWMRIQEum+Wd4eTZA3p2wtQ/XYA7PtuqyvqJiIjMhKU1pDs7tsg3u/vifnqHoCulSffpJ8bivL7d0CXadxuE3U6MiIjIvpjIk+7U6PBodGbMNUWNLhROXfyL152JDu0i8LPfWn4zvrtERETKMZEn3dkwj7e11q3wSkeY6dzUEu9vlmF/q1z6f1co2h4REZGRMZEnA7BfJn882Qzvtd98fh9snDgacx6+OOyYtCL7uB2MYN4tf51dz+3bVeEWiYiIjIuJPOkuwn55vLBhF4f1745+PTrh6sEn4MfHLxOyTtWF0Rs1pkPg/vns7EpERHbBRJ50x9Ka0DW/d5Ik4eIBPdTdlgrrrGlwBr3s368+DV07tQ+4nK/Orq/cpO7cCkRERFpjIk+6s+OEUHZuNW790of17x7Uc375++WYeP1ZQS17SnfvE5E1D4l6+omdg1pPa9eefZLi5xAREamNiTzp7v/bu+8wqaq7D+Df38xsb7C9sLB9WVj60pa2IK6ADRQRC0WxYAUUo9HY4mtM7EZNtUXzGvOqSd68liR2jWg0lmgSjBU11tgQBZVy3j/unWV2du7M3Du3TPl+nmee2b31zJkzM7977inp0rTGTI2vXYF8KmZd6Gs/K848MzNizupZzRhamj9geXZA+7r72bIJmNFajsUThhge47v7j+z3f2GU4S6JiIi8wkCePGfXsIZeWzW9EbevnorjeppjbtvdrM28m0ov3a60ho5U01wRX+24mVPnZfvx0Poe5GX5I65vqijELasm45LFozGipnjA+j+unTlgQrNMvGuU7ApzeHFFRMRAnjwxq62i7+9F4+o8TIl9gu3Uq4tzDbfx+wRTmkqxbs82287ppukt5Qkfw8lWRcHs8PsExXnRAz0RwU1HTBxwR6i92nhoSyuilQeyzp8ut/KIiBLAQJ48cfHi0ThyWiMuWTwaY+oHeZ0c1/xhzQzcdsxUFNhUm2g1jrc6ws0lB42O2GzFaZGC/5P3aB2w7I7V0SaKGqiyOBdLupybXXfO8ErHjk1ERMRAnjxRVZyLc/YdgYMcDKK8EmyLHYnTFeg/X94Vc5uV3Q2WR7ipKcnDw+t70B5lQqZYrPQPiDQ2/OpZTThhdv9mTMNDatOToTnMQVHa4RMRESWKgTyRzRaNq8MgfZjEo2c0unruPeMYXeXQyUMtHTsYGPt8ktAFiZWmNTt2DtwrPzuANXv0b6Lk1IVSKvVlICKizMHeQkQ2y83y409rZ+Kf732O6S3l+Pljbzh2LivxZVsCtel2UBaq5L/8ekfE5XYE2PEkx+ppMniUUSIicgEDeSIHVBbnotKmTo51g/LwzmfbIq/MkKrir3fsimu7ZGhOQ0RE5BY2rSFylUT5b6BxQwfhQIfaWZsdTSX0miG8FrumxLmRWQbnZ6GnvSLiumQP2zN54q+gyqIcr5NARJS2GMgTJZEfHza+3/+xAlWzgezK7oa+v685dFzf360WZjsN9ei3Zse9rdng9t41M5FrMCZ8ODtvUJyiDxHqE2BtAsOFqgxvYHPZkjFeJ4GIKG2xaQ2Ri2IFmvNH1QzcJ8Fz1pfmobmiEFOaynBYSEfXroZSfGfvDvzt35uxbu7AoRyjCQ9Os/w+nLWgA5fd9y98tT2+ZjDxqnawth8wDrSPmdmE+tI8NJYXom5Qnq3HziT1g90frpTct3zqMNz8xJteJ4Mo4zCQJ0pjy6YMwwULOw3XHzWjacCyHx82Hsf997NRjxupVv3omU1YOa0BrWfdG31fG4NbJyfEys3yY9G4xJo1sWkN4EuhfhyFOQF8YdCxmqJLnXeZKL2waQ1REquJURMcK0ayEkPNH1WD61dEH4/eKD7N8sf+SmFwm1l8KfQrc8HCkV4nIWW5Pcs0EWlS6CuWKPXF81N30xET4fcJCnMCOHefETGOF/2IVoPmusHWmpLEw844PtqrT4a4QoEXLslYI5+b5evXzCyootDZZlxERHZj0xqiJNPTXokNZ8xBQU4AhTnRP6JOxUixgk8rY8HHe+xEWMkPs6P3pBIR7y8knArkI832a27/gcuS8JqDiCgq1sgTuSj89rNR4FBVnBsziHdbv+EnvUtGP3YEXqt7mlFTkgu/T/qN5JMOkqE23OdQEr7avtPyvkZ3srzPLSIic5IrUiAiU1Ix8HByJBcrE0LlZwfwyGmzsXnbdlTYPOZ5Incu7OATwGy4e+S0RtzwuD2zERflBlBW6Mw48tt3Jpq3karkEzxkBvO6rBNlKtbIE6WxwQXZXidhALd+7w+fMqzv78UxJtXKDvhsD+KDvAxvrFzYrNvT3FCkRlZ2N+DO47rhd6pKPkG77B0lNeMxjCfyBgN5ohQWqeXExYtHAwDKC7Nx7MyBw0vGI1Kw3S8oTJJf7WhNlVZNb8TK7gYs6RqCsxZ0uJyyJBGhfHQNG4w1exgH60W5Wbac+oTZLWirKrK077WHjkdJXvR0rJtrfZIuI1YufEjj9HwPdmq3WC6JkhGb1hC5yEqYYLaZ85KuekxtKkN5YQ7ysuObEdWsROL48Fvw2QEfvtnRv3rUJ1pnxEsPMjcraOihc7P8OG+/zB5OMFLRueO4buzcpXDVA684e+4EYuK8bB9G1hZjw2sfG25z1IxGfLNzJ6596DVLaYvUxCsJuhSkrCOnNeLiP/zL62TEZcnEelxw1z+9TgaRLVgjT+ShguzErqWNahDrS/MdC+LtdutRk/v9f+PKiXh4/WzceVw3Dhxf51Gq7OF1s2GjwNTt5i6n7Gm+9jxWR92CnABO22u41SRFHLUmLys1PjPJKJd5R+QJ1sgTuSg8gDqupxk3P/Emtm3fie/sbaH5h0PxWKwRc+wcfrKroRR/O7cXAZ9gp1Io1pt2DC3LN33sZKtRVVAYlJeF/2z52pPzezlqTeiZT5zdgsvve9nc/i4nvae9AvWl5sscEZGXbKuRF5EhInKDiLwrIl+LyCYRuVJEBps8znQR+V99/69E5C0RuUdE5tmVViI37T26BgAwqq4EQ8ImWhqUn41HTuvBHaunYtX0xoj7D68udjyN4YaW5WNuR2W/ZXYNPxlp35K8LBTkBPqC+HRy1VLvhrS0GguvnZt4h9fQ/gs+k3cAlHL+IiT8gvLGlRMdPR+5r77UuYntKPnVxZgZPV3YEsiLSDOAZwAcAeApAFcAeB3AGgBPiEhZnMc5DsBjAPbQn68A8AiAWQDuFZGz7EgvkZuuPHgsbj16Mv7n2KkRpzGvLM5FV0Op4RTne42swsKxtRhamj+gGYqToc61h403XJdIk5G9RlZb39lDl5lsrw9o+TSi1v0LsSCrwfDauW144ttzbE5NfMoKsjG9tdyx8eeDwu8qGX3+4hF+gU7JjUNlWnfi7Bavk0Bh7KqR/xGASgAnK6UWKqXOUErNgRaItwO4MNYBRCQLwEUAvgIwQSm1TCn1baXUMgBdAL4GcJaIODNGHJFDsvw+dDeXW26zLiK4cuk4PPqt2ehuKbc5de6Z1FiKKw4eg/Zq50aMcDL2OzDGEJaRODGG+t0nT8fQeJuARMmQ1srCqLvWlCQWnFp5Ly5ePBp3HteNnIA/ocA6FkHkO0NWz3jdiq4EUkNO4ShE9ttzRJXXSaAwCQfyItIEoBfAJgDXhq0+F8CXAJaJSEGMQ5UCKAHwslKqX9d3pdRGAC8DyAMQ/deHKIM4G+yEDe0Y8reVSZ3+a2EnFo0zHwzHcpAeYC8aV4eAPzn67wd8gilNpZjZav+F18jaEhTlJt69KcvhvLJSNJd01aOhXPupsFojP61l9w3gCxZ2GjYTsrNW1ovmbxRbpDKYSCBabMPnLtUlWz8ksqdGPnj/9U9KqX5jyCmltgB4HEA+gCkxjvMhgP8AaBORft+8ItIGoBXA80op4/HIiMgVVmIgp5pKXLx4NDacMQeXLzHf9MUpz52zJ3519BTbLrR6R1ShvDAbN6w0V/Ob2r+55lM/u70C1xwyHqumN+Jb89px2KShKDe4K8LGFekvUqf9Hxw42vLxvGwmR2TEjkC+XX82GpIgOFhx1PHHlFY9coKepmdE5BcicpGI3Ayt/f0/ABxkQ3qJ0oaTgZrdNS9O3T0QEdQOynP07oRZRblZtqbnZ8u78PRZczFnuH23tddZGBLSjESbNVi58MvPDmBwQTbO3mcEju9pgc8nWNJVj8bygTeED5k0tO/vBaO0fhtJVITIBrPaKvr9H/AJSpNwtutUwuZKyceO+0Ql+vNmg/XB5YNiHUgpdbuIvAvgVwCWh6z6AMCN0DrQxiQizxissj7oMFES8irwsFIj72eUlBArFwbR9gkflch2JpMbPlOrpY66EXbJDvhw37qZaDnr3n7LpzSV4aIDRuGNj77EMRZnQKbkZvfIRwxinbGyuwE3bdhkef+q4hx88Lk3Q/wmAzcalAZLfsyffhE5HMD90Eas6YDWJKcDwAMArgFwm0NpJEoZSyfWA9Daa87vrHHsPOE/WYnWMLs9CRFFJyI4cLz9fRas8PsEa8Laso8fFrPuZyCDXxmjvhOHTBqKMxd09DW/yaRAbUy9hfyN4chpkYfQTRZezqtAzjH63GbK6ER2BPLBGvcSg/XFYdtFpLeDvwFaE5plSqmXlFLblFIvAVgGrXnNQSLSEytBSqkJkR4AXorj9RAlte/sMwKXHjQGvz1hmmezt1r5gizOS78x4hPR3RzXqLwJ8TJuMXPuQISLvJXdjZjhQGdhwNlO4qnCievqE2Y3239QOyX4mllsrA10EEui+Rpp/58tm5DYQVOIHYF8cIQZowaXwWqWWNP69QLIAvBIhE6zuwA8qv+bOe8OUQSFOQEsnjAEzRXODuBkR7DTWlmI9qoinL3PCJQwkO9zzMwmXHnwWJy2V3vsjR1k9Uc5J+D8zdzsgA+3rJqMY9nsJSajDr3R2BmT1pTkasdMkUg3QypqHeFE3iV6Jyx8771H1aA3RecrscKONvIP6c+9IuILDcJFpAjANADbADwZ4zjBb6IKg/XB5d9YTSgRWdd/+Mn47DumFifvkfgsoenmzAUdAIATZrfgkj/+K8bW1jkVVsUTr5k5d7TytH6vdvzj3c/x51c/MnFEi1IjDh3ASvy8fac9EdmSriFJ28cgPF9S9O1NKvke3QWOJvwCMnhhmSnXawlXqyilXgPwJwAN0EadCXU+gAIANyulvgwuFJHhIhLe8fQx/XmxiPQbH0pExgJYDO19eTDRNBNRbNF+9OKtlWHN10B5We79EMaqqbVaExbPfmZqZ6M11cry+zBnw3sU+AAAIABJREFUuMMdc5PQyXPin0EzPKf3HVOLW4+aHPWOj13N8i5ePAYtlUUR05FsnGjCkWms3P1xW6a9T3bdHz0e2jjwPxSR3+nDRj4IYB20JjVnhW2/UX/0UUo9BW1kmjwAT4vIbSLyAxH5NYC/AMgFcJVS6h82pZmILIq3ScYuRvKuu/XoyQC0NtBXLR0XdVurTWtSvd9yKiT/lN74m12FBy75WX50t5RH7dxZVZxrNWmuSqaLuFTsDN1eZe9M2oMdGL4z0cA7fIK8VGniZRdbAnm9Vr4LwE0AJgM4FUAzgB8CmGpiEqdVAI4A8ASAvfTj7AngzwAOUUqtsyO9RBSbHd+FDOOdEe296W4ux8Pre/DY6XMcm8Amnh9KU01rTBYUM7XVZqTq779RgBntQvqM+faPxuxE/l10wCjbjpVwW+wULB9ODIiwvtfZOSjMunBRp9dJ8JRt8w0rpd6GFoTHs23Ej4M+KdRN+oOIkkjoj1i0wOuA8XX4zbPvAAAO1ofKTBfzRlbjD/943+tkGFo4thYA0BBhAiQ7xdVG3kTQY/aC7/jZzgTy6WbnLuOcrRuUZ/v5nKixTuTOwcAhdLVnJ0ZesSovy49t23c6dvxvduyKvZFJdt9oTfR4DWX9v+9S8HorIbYF8kSUXqLVukb73j13n5HoqC7GyNpiR4IFL12wsBM7lUJelh8bXvsIH31hvu+9E0HETw4fj5fe34LlUxtsP3Ykdo/HbXY401yH+hmkagAwoFOn/n+6jBRVXZyL9z//qt+yRePq8Nvn3nE1HUMG2/99Nn9UdV/FhxOy/KlaqhMQvGBLnus1R7kxIRQRpYHQL8Wq4t0dnsLHAC/Jz8LRM5vQ3eLMGOBeqijKwc+Xd+GHh4xzfAz//zl2KpZNGYa7Tpoec9t5nTVYO7fNtenn46nxt7N2trwovg52cV8k2ZS02pJcfG+RfU0/rDJ6Oa7fEYuRr+FDid52zJS4DvvLoyYNWDaxoRSn7GmuiUeiF6Cnz0u9yeGPnWXf2P5jhhhNF5SYxDsh9z9AKvZlSAQDeSIydMC4OgBa7ZcvJGC/YslYBHwCEeDmVQN/ZClxkxpLccHCTnTWDfzxPHrG7oDosMlDox5nSlNp39+J/BAX5QSQE/ChKCeASxePjrm9nU1r9h5Vg5G1xfAJkiJwDtrw7T1wqJ7/wU7GXjC6e+bUnQur2quLcMuqSThk0lDcddJ0TGmKb1K04Kg4oSY1lqKs0NyFa6Lh3aB8ey+UJzeWOj7brF2jzExqKMXVh4wHwL5PyYZNa4jI0GVLxuCU3rYBTWRaq4qw4dtz8M2OXRgyON+j1HnLy1qffUfX4v3NX+GTrd/g+J7o7cUvWTwGS3/2JHYphStjjGITzdihg3D1IeOQ5ffBb/OwNZFmdg3l9wn+78Tp+Gzb9qh3HRLvzGh9/+5m83eg9h1Ti/v/+YHtbaS96pQZz3lntFZgRqvRdDHxWTqxHi2VhagvzcONj2/CWx9vxZzhlQP7ryT5QPJFuVlYt2cb7njm346dw66y8D+rp9pzIAcY9YXIFAzkiciQiBgG6pVFqTF8nVNO7W3DmtueBwAMry7CS+9vce3cPp/Efcu8vjQfj35rNpRSCPj734QtNVHDKCJ9NZJf77A38IxnlmKfT2I2HUqmToxBa+e24sr7X4m4rjg3gPtPnYXXPvwCa3/9PD750tp8h5kWuASbDOUE/Pjj2pn48psdKMgO4LfPvYP1t/8t5v7J0nZaJLFOx9ev6EJ2wIcTb30Om7dtj3wOy0c3liz5ZyT4mpPx+8AJbFpDRGTBvqNr8d39R+K0vdpxXI997VCd4PfJgCAeAE4yMeuu9Ps7dnhg5sc+WcZ9diIVa+e24dKDxkRcp6AFcjPbKhI6d5JknycV3n6foDg3C36fYPGEIVG39SqbJjeWIi/Ljwv2H2lreoYMzseM1oqo739OILmaV5kVz6zByVL+vcJAnojIAp9PsHxqA06Y3WJqttbwADdSLfOJLg2vWJKXZekiJJ4fTjO1YW7/Difj735FSIfe7AgXXdFkWuc+q/WsXl0w/uDA0XjxvF4sCxtVKtH28cEWadEumjvrnJlLwi1nLuhAfam5uxaZFtgzkCci8tDNR05CS2UhprWU4YTZzTi+pxnHz3avhj98VkQjoT+O8fxOxgpSQs97cRydZxN19IzGvr9XJ+EdlKuWjoNf70D+iyPNdSAfmNXeRDLJcmclKJlSE+mOWOKjtWjPRsO3LhpX58l7ctT0xtgbmbArxlD44Reywf+TvQmQXdhGnogoQbF+L/YcUYX7/vkBAGDBqJp+6zrrSnD/KbMcSpl9+jWtiRIc+ARYPrVhwIgp01rK8Pir2iTfXcMG4/qVE7Hh1Y8wuCA74sg8dlsztw27FJAd8OHIaZEDDS/j0PbqIjx++hxs37kL9aXmOpBHS/b8zmrc+/fkncSstbIQr3z4hal94n2bwj+XuyeEMu+cfUYYrgvekTPbadlseSsryMbH/fpR6AGrwfZOjSEf627bGfOH47Nt20114j12ZhN++ujrEddFm9gMwIACkWTXk45jIE9E5LALF3Viy1fbEfD5cHaUgCBVhP9OHjOzCY+/+hFOnzccXQ2DkZ898KfliiVjcetTb2FSQ2nfHAPzwy5qrDi4qx6//uvbAGAYoANAYU7As7w3qjENX1xdYq0DebQLq+/u35k0gXykZP748PFYfv1TeHfzVwNXGrDctMbCPkNL8/HTZRPQUWPcRKUkLwu/P3EaJn3vgYjrjdJrtkmUL2x0p9018qYO4zifCC49aAz+a2EnzvzNi/j8qx24f+MHUfdZNnWYYSC/I1Ygn+EYyBMROayyKBe3HZO8w7fFIzRYDA/I1s1tw5kLOqLuX1mci7VzzU3gE48zF3SgZlAuhpXlo6uhNPYOLrpgYacr54kWDlZEmUxrSlMpnnz9E8P1PgGcjqFaKovw2Olz0HzmPbYf246K2aLcQNQgHtA+D5XFFi7CTCYwfJTW4L8Lx9Xil0++Zf78Dgl+P+Rm+XH5wWMBAA1n3B11n0hN8W7Xh7zcGda25tvzh+Oie18acL6+/80mOMWxjTwRUQaLt1Yw3qY1bivJz8LauW1YNC76iCXxsLPT6OTGUhweY7Iu26bWsdi04LIlY7HH8ErD9WYnEzK68xCL3fMSGAkGi27XYBvli9lXPaAtuP56TtvL3Rlnnci/SGV2on5hvmNn/xPGHHo3+D7bkrLkx0CeiChByXZr2wnRgsNMGa/ZjEmNpX2Blt25kx3o/9NtNQyuG5SH61dONFzv9KyjoX5/4jSs7G6w/bihk41FuzsRLzMjVMVi9oLY6HqnJC8r4nKvvpfsvtBn05roGMgTESXIpQrFpJUuFzLu32iweMKw/HbqDomb+TF6yCCct9/I2BuadPOqSRDRav2vXDrW9P7heXDncd0WjhE5I03XyCfJnTAnPu7RLhpjdnYNkxy55B4G8kRECZrVXoEyfTz4WJPSpK70/3m084IkvmNZO+GuGAcPf6fMDmdpdJxUIwJ0N5fjkfWz8fjpczC82vyY6uFZPaJ24DFi5ZPVJkcDzpPqb0gU+dnGdzp2xBh/MjxbkrUTsFMYyBMRJSgn4MfvTpiGq5aOxfkO1Co6Kd7gIHrTGopmbP0gW4+3SykcGtL+funE+n7rw9+rWW0Vls5jtgY4WcvB0LJ8yyMCOclsYJ6unToFQFFuFma0lkdcf82h4/v+vvqQcS6lKnVw1BoiIhvUl+abHv87XdhV45iu2qqKIi63mm3XrejCxIZSdNQUY1RdCQpzAvivuzdG3WdQfhY+27rd1Hm8qAH+yeHjsfqXz+p/T3DkHKZmHbYhD4zONtTk94XZPgup8qkMpnPNHq147JWPBqzfa2Q1rlo6FiIyYB6OSDJtpmMG8kRElJBUCRjcZHcH4EmNpThqeiN8Iuhpq4TPJ1g2ZZit5wjnRSDfO6IaPzl8vP53lfsJsCB8fPdorlvehRNufRZVxbk4vqfF3HmSpW2Nyxfufp9g/7F1cW+/O5sy45uJTWuIiCim8BAidNbIgggTQJG9fAL0jqzG3BFVpgLHxM4Z/TxVxf1Hgcn29w8prDQz8/kE8zprMK+zxvB1xhtHulUze9XS+Jt7zB1Rhae/MxcPre9BXpR24ZFEezvOmO/eEJRWwmOn7q5EkiSXO65hIE9ElMHi/dELDyJ+d8I0rJreiN8e3+3aOOBOc3sYzUmNzkxgFSmAtfIOhe8zraUMbVWFOGJaA/YbU4tbVk3utz43y4/Vs5qRl+XHaXu1Y4UDw0nawc4K5Z8um4DxQ831gSjOzbL0mTHq1AkAq2c14yoLo/K4ZV5nNc4xmFk5Pb49vMNqFCIiMm1kbQlG1pZ4nYyU8f0DRuHCuzeiubIQIkBrZSEWxmgucO6+I3D+//1T/zt67XZRTgBbvt4BAOhqGGxLmsM7u/73UVNi7nPG/OFY39uGgD8z6gknh8wXYLfje5rxo4df6/u/MDfyePFB+4+tw5rbnu/7P9m6rhTlRg85rWZjeP4nSwsktzCQJyKimDKtA5kVK7sbcNOGTQCA5VMb+q1bOmkolnTVm2oWc9jkYSgtyEZlUS46aqIPnXjr0VNw1u9exPDqIuw3ptZs0iOy+o4bBfGZWoasBtThw6dftGgUFvzwscQTFKa8MBsfffFN3NvPbKvA1Q++avo8+4+twxX3vYx3N3/Vb7nd1xvJMt6+WxjIExFRTBn222jJ+r3a0VJZiI6aYlQVDxzu0Gzb9uyAL+5OfqOGlOD3J043dfxYUv09dzv9TeUFeP2jL207XnhTr6aKgn7/23dhZO44ExtKccb84fj+vS+Z2i874MMDp/bg7U+3oveKR03tS8Yy494XERFRDHlZftQmMN54YU4Ah08ZhgnD7GnaYicrtZSN5QWxN/JA/eC8uLYbNcT5pl+hte1nG7QBt35wew9nJNb15RHTGgYsWz2rGTcfOQl7j6ox1TY/L9s/YDhWp663kq1pkVMYyBMRUUypXjsbDxHBrUfHbgeezlZ2N6C0IBt7jazCHh32Dv+YlUC7+VtWTcKkxlJcsP9IVEa42xF053FT0VlXjOVTh6HH4kRYVpXqszvbpae9st//Zj+D8XbejnbcY2Y2Yd2ebRHXzWyrwLWHjUd3c+SJnNwSrRNwJmDTGiKiDBb3zK4Z0r65IUlrod3SO6IK5+wzAj6f4Lan3kr4eKumN+L6P7+BquIc9I60fmEwo7UCM1pjB+YThpXirpNmWD5PODPlvrWq0GBN/FXDZy3owPP//gydtSWY2lyGmW0VePTl/2BKUylyAuaGqww6f7+ROPf3/zBcbzTMaE97Bc5c0GHpnG5bMXUYfvHEm8j2+3DIxKGxd0gjDOSJiCi2zIjjSXa35Q+tER5ncojFoDMXdGBuRxVG1BQnVCOfrELHgs/PDuDO46bi/o0f4scho82Y0VhegKNnNvX9//PlE/DMpk8xPkJzrexAfPm5orsBnXUlOPDHGyKujxTI+32CC/bvjDPV3jtt3nB01BSjs64Eg/U7IxnSsoaBPBERkZGV3Q34dOs3GDI4D395/RP89c1PvU6SJVauw6pLcnHjyol48o2PB4zCEy+/TzC1uczSvk5RCTaevuLgMfjpI6/jsMlDkZvVv5Z8wrBSTBhWajmQD5cT8KO7ZXfTlWVThuGWJ99ET3sFqk305yjMMQ73wuP4e06egUH5WagdFF9fhETt7stgvbagMCeApZMyqyY+iIE8ERHFlKkV8jUluThPn6H0qvtf6QvkmyvSswlOeFOS2cMrMXt4pcHWmWnRuCFYNG5I1G0mNZbiqTc+QXlhDhrK7Csr391/JI6e0YT60jiCbBX6p/HFS3iN/Ija6EOdhrPSJv321VNx+h0vYGRdCfYdrQ2XmhPnHQY7zp9OGMgTEWWweNsAF+dFn4wmExw7qwkvvvMZPtzyNS5fkryzaJL3rjl0HO554T3MbKswNTlWrBlfRQRDy/ITTV7YMRPb38oNjokNpXhwfU+/ZSNrizGqrgQvvrMZK6YOSyxRSPzOS6pgIE9ElMGMO+gBFx84Gt+68wXkZvlwqsHIFeluaOnuoCk3y4/rVkz0MDVkBzfiu8qiXKyc1hjXtodMqsevnnob1cW5mNHq/ggwyVKhLSL4zfHdePXDLzC8uijqttNayvD4qx9j3NBByM/O7FA2s189EVGGm9VWgQPHD8Ff3vgYFy4a1W/dkon16KwrQVVxDsoKczxKoft+cvgEnHbH3zBh2GDM66z2OjmuyvRmCl44f79O9I6oxughJaZq782IdvFiNGqNF7L8vpizGAPAz5Z14fFXP0q6/hdeYCBPRJTBRASXLRljuN5se9l0MK+zGnM7Kh0LqpJZ8oR0mSM74HOkH0Jo7B4tkD99/nAce8szAIBTUuTOW0FOAL0jM+si2wgDeSIiojDpFsTHPV9AEtXOJoNMyI65HVW4ZPFobN62HYdNTrxtOrmLgTwRERFRGos2ao0AOKir3r3EuCQzuroC6VXlQERERJZlQg10pADvSL1j6oQIEy+lg2hNazLhPU9nrJEnIiIiAJnbRv5b89pxxLQG1A3Kw/o7/obfPPsOAG0CplQV7/CLbE6V2hjIExERpT0Ga7HU60ONnr33CBTlBFBakIMDxkef+ClV7T26Bne/8B72GV2T8LFysti4w0sM5ImIiChjZMeYQXRwQTbO37/TpdR445pDxuHUPdvQWJ74rLPFuVk4dPJQ/Prpt3F8T7MNqbNHhswHxTbyREREpMmEVhaHTxmGgmx/v2Xp/rrDg1oRQVNFoW3Nar63aBRePK8Xp/a223I8ih8DeSIiItKleUQLoDAngAdO7em3TDLgdTst02dY9QoDeSIiojSX7jXOZlUV95+pOB3zp9+EUBkzGONu8Xb2TXUM5ImIiAhAega0kYQ3KUn3l50hMW1GYiBPREREANI/oDXCIRgpVTGQJyIioozGMD79FOVmeZ0EVzCQJyIiSnPxBqqZVDO9bm4bCrL9WLNHK3y+9Hvdoc1pMrFlzbWHjfc6Ca5gF2MiIiICkFk102vmtuLEOS3wp2EQHy5TOn6GGls/CAd31ePXf33b66Q4ijXyREREGWxuRxUAoKYkFyNriz1OjbsyIYjPZK1VhV4nwXEM5ImIiDLYJYtH43uLRuG2Y6Yg4GdYkI4yrz5eM39UTd/f01vKPUyJc9i0hoiIKM1Fa/o+uCAbh04e6l5iyBX9xpHP0Ei+blAebjxiIp5981McPmWY18lxBC+9iYiI0tzxPS19fx81vdHDlBC5a3Z7JU7tbUdVca7XSXEEa+SJiIjS3KGTh+Kzrdux9ZsdOHFOS+wdiCglMJAnIiJKc1l+H9bMbfU6GeSZDG1bkwHYtIaIiIgojWVqG/lMwECeiIiIiCgFMZAnIiIiSjOhk0CxQj59MZAnIiIiSmNsWpO+GMgTERERpbGAnzPYpisG8kRERERpbOyQQWgoywcAHDRhiMepITtx+EkiIiKiNBPamsbnE/z+pOl44e3NmNxU6lmayH4M5ImIiIjSXHFuFqa3lnudDLIZm9YQERERpZniXNbVZgIG8kRERERp4OIDRwMAcrN8OLW33ePUkBt4uUZERESUBpZMrEdnXQkqi3NQXpjjdXLIBQzkiYiIiNLEiNpir5NALmLTGiIiIiKiFMRAnoiIiIgoBTGQJyIiIiJKQQzkiYiIiIhSEAN5IiIiIqIUxECeiIiIiCgFMZAnIiIiIkpBDOSJiIiIiFKQbYG8iAwRkRtE5F0R+VpENonIlSIy2MKxRonIzSLytn6sD0XkERFZbld6iYiIiIhSmS0zu4pIM4ANACoB/C+AlwBMArAGwDwRmaaU+jjOY60EcB2ArQDuArAJwCAAnQAWALjZjjQTEREREaUyWwJ5AD+CFsSfrJS6OrhQRC4HsA7AhQBWxzqIiEyBFsT/HcA8pdT7YeuzbEovEREREVFKS7hpjYg0AeiFVnN+bdjqcwF8CWCZiBTEcbiLAfgBHB4exAOAUmp7YqklIiIiIkoPdtTIz9Gf/6SU2hW6Qim1RUQehxboTwHwgNFBRGQIgBkA/grgHyIyG8AEAArA8wAeCj8+EREREVGmsiOQb9efXzZY/wq0QL4NUQJ5ABNDtn8QQE/Y+hdF5ACl1KuxEiQizxisGh5rXyIiIiKiVGDHqDUl+vNmg/XB5YNiHKdSf14CoAPAAfqxWwDcAmAUgLtFJNt6UomIiIiI0oNdnV2jEf1ZxdjOH/J8lFLqLv3/z0VkBbTgvgvAgQB+Fe1ASqkJEROi1dSPjyfRRERERETJzI5APljjXmKwvjhsOyOf6s9fA7gndIVSSonI/0IL5CchRiAfRcPGjRsxYULEOJ+IiIiIyBYbN24EgAYnz2FHIP8v/bnNYH2r/mzUhj78OFsMOrUGA/08E2kL9/m2bdvw7LPPbkrgGFYF2+e/5MG5UxnzzRrmmzXMN2uYb9Yw38xjnlnDfLMm0XxrAPC5PUmJzI5A/iH9uVdEfKFBuIgUAZgGYBuAJ2Mc5wUAHwEoF5EqpdQHYes79edNVhOqlGq0um+igh1wjZr9UGTMN2uYb9Yw36xhvlnDfDOPeWYN882aVMi3hDu7KqVeA/AnaFcdJ4StPh9AAYCblVJfBheKyHAR6TeCjFJqB4Cf6v9eLCK+kO1HAVgJYAeAOxJNMxERERFRqrOrs+vxADYA+KGI7AFgI4DJAGZDa1JzVtj2G/VnCVv+PQB7AFgOYJSIPAygAloH11wAp8Yz/CQRERERUbqzY/jJYK18F4CboAXwpwJoBvBDAFOVUh/HeZyt0AL58wHkQ6vh3w/aRcICpdTldqSXiIiIiCjV2Tb8pFLqbQBHxLlteE186LqtAM7TH0REREREFIEtNfJEREREROQuUSrWPE1ERERERJRsWCNPRERERJSCGMgTEREREaUgBvJERERERCmIgTwRERERUQpiIE9ERERElIIYyBMRERERpSAG8kREREREKYiBvMNEZIiI3CAi74rI1yKySUSuFJHBXqfNDfrrVQaP9w326RaRe0TkExHZKiIviMhaEfFHOc8+IvKwiGwWkS9E5C8issK5V5Y4EVksIleLyGMi8rmeJ7+MsY8reSMiK0TkKX37zfr++1h9rXYyk28i0hCl/CkRuS3KeUzlgYj49ffiBRHZpr9H94hItx2vOxEiUiYiR4nIb0XkVT19m0XkzyKySkQi/hZkenkzm28sb7uJyA9E5AEReTskfc+JyLkiUmawT0aXN8BcvrG8RSciy0Ly4iiDbRwvP47nnVKKD4ceAJoBfABAAfgdgO8DeFD//yUAZV6n0YU82ATgMwDnRXisj7D9/gB2APgCwPUALtHzSgG43eAcJ+rrPwJwLYArALytL7vU6zyIkjfP62ncAmCj/vcvo2zvSt4AuFRf/7a+/bUAPtaXnZhK+QagQV//vEEZXGxHHgAQALeHfLYv0d+jL/T3bH+P82y1nrZ3Afw3gIsA3KB/NhWAO6BPEMjyZj3fWN76pfEbAE/q+fV9AFcDeFpP8zsA6lneEss3lreo+Vivf0636Ok+yovy40beeZ7Z6fwA8Ef9zTspbPnl+vKfeJ1GF/JgE4BNcW5bDOBDAF8D6ApZngtgg55nS8P2aQDwlf5BaghZPhjAq/o+U73OB4PXOxtAq/5B70H0gNSVvAHQrS9/FcDgsGN9rB+vIZHX7XK+NejrbzJxfNN5AOAQfZ/HAeSGLJ+ov2cfAijyMM/mANgXgC9seTWAt/S0H8jylnC+sbyFlBWD5Rfqaf8Ry1vC+cbyFvk1CoD7AbwGLXAeEMi7VX7cyDvPMzxdHwCa9DfvDQz8ESiCdjX2JYACr9PqcD5sQvyB/JF6nv0iwro5+rpHwpZ/V19+vpnjJdsDsQNSV/IGwM368iMi7GN4vCTOtwaY/6EznQcAHtWXzzZzvGR4ADhTT9/VLG8J5xvLW+zXO0ZP330sbwnnG8tb5Ne4BsAuADOh3ZmIFMi7Un7cyDu2kXfOHP35T0qpXaErlFJboF2d5QOY4nbCPJAjIoeLyJkiskZEZhu0eQzm2R8irHsUwFYA3SKSE+c+94Ztk8rcypt0zc9aETlWL4PHisjoKNuaygM9z7uhvQePxbNPktmuP+8IWcbyFlukfAtieTO2r/78QsgylrfYIuVbEMubTkQ6oDVJukop9WiUTR0vP27lXSCRnSmqdv35ZYP1rwDoBdAG4AFXUuSdagC3hC17Q0SOUEo9ErLMMM+UUjtE5A0AI6Hd7dgYxz7viciXAIaISL5SamsiL8JjjueNiBQAqAPwhVLqvQhpeEV/bkvgdXhlT/3RR0QeBrBCKfVWyDIredACwA/gdaVUpKAuafNNRAIAluv/hv44sbxFESXfgljedCKyHkAhgBIAXQCmQwtGvx+yGctbmDjzLYjlDX2fy1ugNXs7M8bmbpQfV/KONfLOKdGfNxusDy4f5EJavHQjgD2gBfMFAEYB+Cm0W4L3isiYkG2t5Fm8+5QYrE8VbuRNOpbZrQAuADABWtvHwQBmAXgIWrOcB/Qv6CAn8zkZ8+37ADoB3KOU+mPIcpa36IzyjeVtoPUAzgWwFlow+gcAvUqp/4Rsw/I2UDz5xvLW3zkAxgFYqZTaFmNbN8qPK3nHQN47oj8rT1PhMKXU+UqpB5VSHyiltiql/q6UWg2tw28etPZr8bKSZxmRz3A3b1ImL5VSHyqlzlFKPauU+kx/PArtbthfoNWYRByWLNahTWyblGVQRE4GcCq0kRSWmd1df8648hYt31jeBlJKVSulBFplzgHQatWfE5HxJg6TceUtnnxjeQtJhMgkaLXwlymlnrDjkPqzk+XHlrxjIO+cWDXBxWHbZZqf6M8zQ5ZZybN49/ncVOqSjxvq8cdEAAAEPUlEQVR5E2v7WLULKUO/zXmd/q+ZMhgpD1Lusy4iJwC4CsA/oXXC+iRsE5a3COLIt4gyvbwBgF6Z81toQWYZtI5+QSxvBmLkm9E+GVXeQprUvAzg7Dh3c6P8uJJ3DOSd8y/92ajtU6v+bNSGPt19qD+H3vYzzDP9g9oIrWPZ63HuU6Mf/98p3j4ecCFvlFJfQhunuFBfHy7dymzwFnVfGbSYB68C2AmgSX8v4tnHMyKyFsA1AP4OLRiNNDEby1uYOPMtmowsb+GUUm9CuxAaKSLl+mKWtxgM8i2aTCpvhdDKQQeAr0ImgVLQmicBwM/1ZVfq/7tRflzJOwbyznlIf+6VgbP/FQGYBmAbtIkfMtFU/Tn0i/lB/XlehO1nQhvlZ4NS6us495kftk0qcytvMiU/gd0jRr0ettxUHuh5vgHaezAjnn28IiKnQ5vE5HloweiHBpuyvIUwkW/RZFx5i6JWf96pP7O8xSc836LJpPL2NbRJliI9ntO3+bP+f7DZjePlx7W8S2TsSj5ijmWa0RNCQRtloDTC8mHQemsrAGeGLC+GVotgZlKQRqTohFBhr6MH0cdDdyVvkAITppjMt8kAsiMsn6O/FgWgO9E8QHyTfhR7nFdn62n8a6TPJcubLfnG8qalYziA6gjLfdg9sdHjLG8J5xvLW+w8PQ+Rx5F3pfy4kXeeZ3I6PwA0A/hAfxN/B2167wf1//8FoMzrNDr8+s/TC/a9AH4E4AfQpjTfpufB3eFfQgAWYvc03dcBuBgh03QjbBp5fZ+T9PVxT7OcDA/9td6kP/6gp/e1kGWXRtje8bwBcJm+PnQK6o/0ZckwhXnc+QbgYWgBwu36a7kC2nCvSn98x448QP9puDfq703STGEOYIWeth366zkvwmMly1ti+cby1pe+tdDG2X8AwM+g/fbdAO1zqgC8B2AEy1ti+cbyFleenocIgbxb5ceNvPM8k9P9AaAe2hCM7wH4BsCb0DpLRa3ZSYcHtGGwfqV/GX+mf0H9B8B90MZgHvDFrO83DcA9AD6FFvS/CGAdAH+Uc+0L4BEAW6DNmPs0tDF0Pc+HKGkOfsEYPTZ5lTfQApin9e236Pvv43Wemc03AKsA3AVthuEvoNWAvAXg1wBm2JkH0OblWKe/J9v09+gehNWIJWmeKQAPs7wllm8sb31p64QW4DwPLcjZAa1D39N6nkb8/WN5M5dvLG9x5WnwMzwgkHer/Didd6KfhIiIiIiIUgg7uxIRERERpSAG8kREREREKYiBPBERERFRCmIgT0RERESUghjIExERERGlIAbyREREREQpiIE8EREREVEKYiBPRERERJSCGMgTEREREaUgBvJERERERCmIgTwRERERUQpiIE9ERERElIIYyBMRERERpSAG8kREREREKYiBPBERERFRCmIgT0RERESUghjIExERERGloP8HPT+e1RHFzEEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 377
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses['test'], label='Test loss')\n",
    "plt.legend()\n",
    "_ = plt.ylim()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tensors(loaded_graph):\n",
    "\n",
    "    uid = loaded_graph.get_tensor_by_name(\"uid:0\")\n",
    "    user_gender = loaded_graph.get_tensor_by_name(\"user_gender:0\")\n",
    "    user_age = loaded_graph.get_tensor_by_name(\"user_age:0\")\n",
    "    user_job = loaded_graph.get_tensor_by_name(\"user_job:0\")\n",
    "    movie_id = loaded_graph.get_tensor_by_name(\"movie_id:0\")\n",
    "    movie_categories = loaded_graph.get_tensor_by_name(\"movie_categories:0\")\n",
    "    movie_titles = loaded_graph.get_tensor_by_name(\"movie_titles:0\")\n",
    "    targets = loaded_graph.get_tensor_by_name(\"targets:0\")\n",
    "    dropout_keep_prob = loaded_graph.get_tensor_by_name(\"dropout_keep_prob:0\")\n",
    "    lr = loaded_graph.get_tensor_by_name(\"LearningRate:0\")\n",
    "    #两种不同计算预测评分的方案使用不同的name获取tensor inference\n",
    "#     inference = loaded_graph.get_tensor_by_name(\"inference/inference/BiasAdd:0\")\n",
    "    inference = loaded_graph.get_tensor_by_name(\"inference/ExpandDims:0\") \n",
    "    movie_combine_layer_flat = loaded_graph.get_tensor_by_name(\"movie_fc/Reshape:0\")\n",
    "    user_combine_layer_flat = loaded_graph.get_tensor_by_name(\"user_fc/Reshape:0\")\n",
    "    return uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob, inference, movie_combine_layer_flat, user_combine_layer_flat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "指定用户和电影进行评分\n",
    "这部分就是对网络做正向传播，计算得到预测的评分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rating_movie(user_id_val, movie_id_val):\n",
    "    loaded_graph = tf.Graph()  #\n",
    "    with tf.Session(graph=loaded_graph) as sess:  #\n",
    "        # Load saved model\n",
    "        loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "        loader.restore(sess, load_dir)\n",
    "    \n",
    "        # Get Tensors from loaded model\n",
    "        uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob, inference,_, __ = get_tensors(loaded_graph)  #loaded_graph\n",
    "    \n",
    "        categories = np.zeros([1, 18])\n",
    "        categories[0] = movies.values[movieid2idx[movie_id_val]][2]\n",
    "    \n",
    "        titles = np.zeros([1, sentences_size])\n",
    "        titles[0] = movies.values[movieid2idx[movie_id_val]][1]\n",
    "    \n",
    "        feed = {\n",
    "              uid: np.reshape(users.values[user_id_val-1][0], [1, 1]),\n",
    "              user_gender: np.reshape(users.values[user_id_val-1][1], [1, 1]),\n",
    "              user_age: np.reshape(users.values[user_id_val-1][2], [1, 1]),\n",
    "              user_job: np.reshape(users.values[user_id_val-1][3], [1, 1]),\n",
    "              movie_id: np.reshape(movies.values[movieid2idx[movie_id_val]][0], [1, 1]),\n",
    "              movie_categories: categories,  #x.take(6,1)\n",
    "              movie_titles: titles,  #x.take(5,1)\n",
    "              dropout_keep_prob: 1}\n",
    "    \n",
    "        # Get Prediction\n",
    "        inference_val = sess.run([inference], feed)  \n",
    "    \n",
    "        return (inference_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成Movie特征矩阵\n",
    "将训练好的电影特征组合成电影特征矩阵并保存到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n"
     ]
    }
   ],
   "source": [
    "loaded_graph = tf.Graph()  #\n",
    "movie_matrics = []\n",
    "with tf.Session(graph=loaded_graph) as sess:  #\n",
    "    # Load saved model\n",
    "    loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "    loader.restore(sess, load_dir)\n",
    "\n",
    "    # Get Tensors from loaded model\n",
    "    uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob, _, movie_combine_layer_flat, __ = get_tensors(loaded_graph)  #loaded_graph\n",
    "\n",
    "    for item in movies.values:\n",
    "        categories = np.zeros([1, 18])\n",
    "        categories[0] = item.take(2)\n",
    "\n",
    "        titles = np.zeros([1, sentences_size])\n",
    "        titles[0] = item.take(1)\n",
    "\n",
    "        feed = {\n",
    "            movie_id: np.reshape(item.take(0), [1, 1]),\n",
    "            movie_categories: categories,  #x.take(6,1)\n",
    "            movie_titles: titles,  #x.take(5,1)\n",
    "            dropout_keep_prob: 1}\n",
    "\n",
    "        movie_combine_layer_flat_val = sess.run([movie_combine_layer_flat], feed)  \n",
    "        movie_matrics.append(movie_combine_layer_flat_val)\n",
    "\n",
    "pickle.dump((np.array(movie_matrics).reshape(-1, 200)), open('movie_matrics.p', 'wb'))\n",
    "movie_matrics = pickle.load(open('movie_matrics.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "movie_matricsmovie_ma  = pickle.load(open('movie_matrics.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成User特征矩阵\n",
    "将训练好的用户特征组合成用户特征矩阵并保存到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n"
     ]
    }
   ],
   "source": [
    "loaded_graph = tf.Graph()  #\n",
    "users_matrics = []\n",
    "with tf.Session(graph=loaded_graph) as sess:  #\n",
    "    # Load saved model\n",
    "    loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "    loader.restore(sess, load_dir)\n",
    "\n",
    "    # Get Tensors from loaded model\n",
    "    uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob, _, __,user_combine_layer_flat = get_tensors(loaded_graph)  #loaded_graph\n",
    "\n",
    "    for item in users.values:\n",
    "\n",
    "        feed = {\n",
    "            uid: np.reshape(item.take(0), [1, 1]),\n",
    "            user_gender: np.reshape(item.take(1), [1, 1]),\n",
    "            user_age: np.reshape(item.take(2), [1, 1]),\n",
    "            user_job: np.reshape(item.take(3), [1, 1]),\n",
    "            dropout_keep_prob: 1}\n",
    "\n",
    "        user_combine_layer_flat_val = sess.run([user_combine_layer_flat], feed)  \n",
    "        users_matrics.append(user_combine_layer_flat_val)\n",
    "\n",
    "pickle.dump((np.array(users_matrics).reshape(-1, 200)), open('users_matrics.p', 'wb'))\n",
    "users_matrics = pickle.load(open('users_matrics.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_matrics = pickle.load(open('users_matrics.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "开始推荐电影\n",
    "使用生产的用户特征矩阵和电影特征矩阵做电影推荐\n",
    "\n",
    "推荐同类型的电影\n",
    "思路是计算当前看的电影特征向量与整个电影特征矩阵的余弦相似度，取相似度最大的top_k个，这里加了些随机选择在里面，保证每次的推荐稍稍有些不同。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommend_same_type_movie(movie_id_val, top_k = 20):\n",
    "    \n",
    "    loaded_graph = tf.Graph()  #\n",
    "    with tf.Session(graph=loaded_graph) as sess:  #\n",
    "        # Load saved model\n",
    "        loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "        loader.restore(sess, load_dir)\n",
    "        \n",
    "        norm_movie_matrics = tf.sqrt(tf.reduce_sum(tf.square(movie_matrics), 1, keep_dims=True))\n",
    "        normalized_movie_matrics = movie_matrics / norm_movie_matrics\n",
    "\n",
    "        #推荐同类型的电影\n",
    "        probs_embeddings = (movie_matrics[movieid2idx[movie_id_val]]).reshape([1, 200])\n",
    "        probs_similarity = tf.matmul(probs_embeddings, tf.transpose(normalized_movie_matrics))\n",
    "        sim = (probs_similarity.eval())\n",
    "    #     results = (-sim[0]).argsort()[0:top_k]\n",
    "    #     print(results)\n",
    "        \n",
    "        print(\"您看的电影是：{}\".format(movies_orig[movieid2idx[movie_id_val]]))\n",
    "        print(\"以下是给您的推荐：\")\n",
    "        p = np.squeeze(sim)\n",
    "        p[np.argsort(p)[:-top_k]] = 0\n",
    "        p = p / np.sum(p)\n",
    "        results = set()\n",
    "        while len(results) != 5:\n",
    "            c = np.random.choice(3883, 1, p=p)[0]\n",
    "            results.add(c)\n",
    "        for val in (results):\n",
    "            print(val)\n",
    "            print(movies_orig[val])\n",
    "        \n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n",
      "您看的电影是：[1202 'Withnail and I (1987)' 'Comedy']\n",
      "以下是给您的推荐：\n",
      "1184\n",
      "[1202 'Withnail and I (1987)' 'Comedy']\n",
      "546\n",
      "[550 'Threesome (1994)' 'Comedy|Romance']\n",
      "2822\n",
      "[2891 'Happy, Texas (1999)' 'Comedy']\n",
      "2153\n",
      "[2222 'Champagne (1928)' 'Comedy']\n",
      "1617\n",
      "[1663 'Stripes (1981)' 'Comedy']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{546, 1184, 1617, 2153, 2822}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommend_same_type_movie(1202, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "推荐您喜欢的电影\n",
    "思路是使用用户特征向量与电影特征矩阵计算所有电影的评分，取评分最高的top_k个，同样加了些随机选择部分。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommend_your_favorite_movie(user_id_val, top_k = 10):\n",
    "\n",
    "    loaded_graph = tf.Graph()  #\n",
    "    with tf.Session(graph=loaded_graph) as sess:  #\n",
    "        # Load saved model\n",
    "        loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "        loader.restore(sess, load_dir)\n",
    "\n",
    "        #推荐您喜欢的电影\n",
    "        probs_embeddings = (users_matrics[user_id_val-1]).reshape([1, 200])\n",
    "\n",
    "        probs_similarity = tf.matmul(probs_embeddings, tf.transpose(movie_matrics))\n",
    "        sim = (probs_similarity.eval())\n",
    "    #     print(sim.shape)\n",
    "    #     results = (-sim[0]).argsort()[0:top_k]\n",
    "    #     print(results)\n",
    "        \n",
    "    #     sim_norm = probs_norm_similarity.eval()\n",
    "    #     print((-sim_norm[0]).argsort()[0:top_k])\n",
    "    \n",
    "        print(\"以下是给您的推荐：\")\n",
    "        p = np.squeeze(sim)\n",
    "        p[np.argsort(p)[:-top_k]] = 0\n",
    "        p = p / np.sum(p)\n",
    "        results = set()\n",
    "        while len(results) != 5:\n",
    "            c = np.random.choice(3883, 1, p=p)[0]\n",
    "            results.add(c)\n",
    "        for val in (results):\n",
    "            print(val)\n",
    "            print(movies_orig[val])\n",
    "\n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n",
      "以下是给您的推荐：\n",
      "2912\n",
      "[2981 'Brother, Can You Spare a Dime? (1975)' 'Documentary']\n",
      "1650\n",
      "[1697 'Big Bang Theory, The (1994)' 'Crime']\n",
      "1939\n",
      "[2008 'This World, Then the Fireworks (1996)' 'Crime|Drama|Film-Noir']\n",
      "3314\n",
      "[3383 'Big Fella (1937)' 'Drama|Musical']\n",
      "315\n",
      "[318 'Shawshank Redemption, The (1994)' 'Drama']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{315, 1650, 1939, 2912, 3314}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommend_your_favorite_movie(24, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "看过这个电影的人还看了（喜欢）哪些电影\n",
    "首先选出喜欢某个电影的top_k个人，得到这几个人的用户特征向量。\n",
    "然后计算这几个人对所有电影的评分\n",
    "选择每个人评分最高的电影作为推荐\n",
    "同样加入了随机选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def recommend_other_favorite_movie(movie_id_val, top_k = 20):\n",
    "    loaded_graph = tf.Graph()  #\n",
    "    with tf.Session(graph=loaded_graph) as sess:  #\n",
    "        # Load saved model\n",
    "        loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "        loader.restore(sess, load_dir)\n",
    "\n",
    "        probs_movie_embeddings = (movie_matrics[movieid2idx[movie_id_val]]).reshape([1, 200])\n",
    "        probs_user_favorite_similarity = tf.matmul(probs_movie_embeddings, tf.transpose(users_matrics))\n",
    "        favorite_user_id = np.argsort(probs_user_favorite_similarity.eval())[0][-top_k:]\n",
    "    #     print(normalized_users_matrics.eval().shape)\n",
    "    #     print(probs_user_favorite_similarity.eval()[0][favorite_user_id])\n",
    "    #     print(favorite_user_id.shape)\n",
    "    \n",
    "        print(\"您看的电影是：{}\".format(movies_orig[movieid2idx[movie_id_val]]))\n",
    "        \n",
    "        print(\"喜欢看这个电影的人是：{}\".format(users_orig[favorite_user_id-1]))\n",
    "        probs_users_embeddings = (users_matrics[favorite_user_id-1]).reshape([-1, 200])\n",
    "        probs_similarity = tf.matmul(probs_users_embeddings, tf.transpose(movie_matrics))\n",
    "        sim = (probs_similarity.eval())\n",
    "    #     results = (-sim[0]).argsort()[0:top_k]\n",
    "    #     print(results)\n",
    "    \n",
    "    #     print(sim.shape)\n",
    "    #     print(np.argmax(sim, 1))\n",
    "        p = np.argmax(sim, 1)\n",
    "        print(\"喜欢看这个电影的人还喜欢看：\")\n",
    "\n",
    "        results = set()\n",
    "        while len(results) != 2:\n",
    "            c = p[random.randrange(top_k)]\n",
    "            results.add(c)\n",
    "        for val in (results):\n",
    "            print(val)\n",
    "            print(movies_orig[val])\n",
    "        \n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n",
      "您看的电影是：[201 'Three Wishes (1995)' 'Drama']\n",
      "喜欢看这个电影的人是：[[2154 'M' 25 12]\n",
      " [5763 'F' 25 1]\n",
      " [5669 'M' 56 1]\n",
      " [1669 'F' 25 17]\n",
      " [2202 'F' 1 10]\n",
      " [5155 'M' 25 16]\n",
      " [3008 'M' 18 4]\n",
      " [2292 'M' 50 15]\n",
      " [1763 'M' 35 7]\n",
      " [3031 'M' 18 4]\n",
      " [100 'M' 35 17]\n",
      " [2391 'M' 50 18]\n",
      " [1568 'F' 1 10]\n",
      " [3603 'F' 35 7]\n",
      " [3901 'M' 18 14]\n",
      " [3764 'M' 25 1]\n",
      " [5861 'F' 50 1]\n",
      " [4085 'F' 25 6]\n",
      " [85 'M' 18 4]\n",
      " [2338 'M' 45 17]]\n",
      "喜欢看这个电影的人还喜欢看：\n",
      "2912\n",
      "[2981 'Brother, Can You Spare a Dime? (1975)' 'Documentary']\n",
      "1650\n",
      "[1697 'Big Bang Theory, The (1994)' 'Crime']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{1650, 2912}"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommend_other_favorite_movie(201, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以上就是实现的常用的推荐功能，将网络模型作为回归问题进行训练，得到训练好的用户特征矩阵和电影特征矩阵进行推荐。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
